fix(backups): file level restore : omnibus fixes (#9776)

@vates/fuse-vhd/index.mjs

@@ -1,12 +1,12 @@
-import LRU from 'lru-cache'
 import Fuse from 'fuse-native'
+import LRU from 'lru-cache'
 import { createLogger } from '@xen-orchestra/log'
 import { VhdSynthetic } from 'vhd-lib'
 import { Disposable, fromCallback } from 'promise-toolbox'
 
 const { warn } = createLogger('vates:fuse-vhd')
 
-// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
+// build a stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
   mtime: st.mtime || new Date(),
   atime: st.atime || new Date(),
@@ -20,42 +20,109 @@ const stat = st => ({
 export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
   const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
 
-  const cache = new LRU({
-    max: 16, // each cached block is 2MB in size
-  })
   await vhd.readBlockAllocationTable()
+  const { blockSize } = vhd.header
+
+  // A zeroed buffer returned for VHD blocks that are not present in the chain.
+  // Never mutated — safe to share across all reads.
+  const EMPTY_BLOCK = Buffer.alloc(blockSize, 0)
+
+  // --- coalescing block read queue ---
+  //
+  // FUSE read requests from NTFS-3g arrive in small chunks (typically 4 KiB)
+  // while VHD blocks are 2 MiB. Reading blocks one at a time from the backing
+  // store (CIFS/NFS/…) bounds libuv threadpool usage to 1 concurrent I/O call.
+  //
+  // Coalescing: all pending requests for the same blockId share a single
+  // readBlock() call. When the block resolves, every waiter is fulfilled at once.
+  //
+  // Together these two properties prevent the FUSE-on-FUSE threadpool deadlock:
+  // concurrent NTFS-3g reads no longer starve the VHD block reads they depend on.
+  // In-memory block cache. Each VHD block is 2 MiB; 16 blocks = 32 MiB.
+  // This is the primary tool against the 300× CIFS amplification: NTFS-3g
+  // reads a 2 MiB block in ~512 sequential 4 KiB FUSE requests. Without a
+  // cache, each request causes a full 2 MiB CIFS read. With the cache, only
+  // the first request reads from CIFS; the rest are served from RAM.
+  const blockCache = new LRU({ max: 64 })
+
+  const pendingBlocks = new Map() // blockId → Promise<Buffer>
+  const readQueue = [] // { blockId, resolve, reject }[]
+  let queueRunning = false
+
+  // log stats every 10 s while there is activity
+
+  function enqueueBlock(blockId) {
+    // 1. serve from cache if available
+    if (blockCache.has(blockId)) {
+      return Promise.resolve(blockCache.get(blockId))
+    }
+    // 2. coalesce: join an existing in-flight read for this block
+    if (pendingBlocks.has(blockId)) {
+      return pendingBlocks.get(blockId)
+    }
+    const p = new Promise((resolve, reject) => {
+      readQueue.push({ blockId, resolve, reject })
+    })
+    pendingBlocks.set(blockId, p)
+    p.then(
+      data => {
+        pendingBlocks.delete(blockId)
+        blockCache.set(blockId, data)
+      },
+      () => pendingBlocks.delete(blockId)
+    )
+    drainQueue()
+    return p
+  }
+
+  function drainQueue() {
+    if (queueRunning) return
+    queueRunning = true
+    // async IIFE — errors are forwarded to individual reject() callbacks,
+    // so the outer floating promise is always fulfilled
+    ;(async () => {
+      while (readQueue.length > 0) {
+        const { blockId, resolve, reject } = readQueue.shift()
+        try {
+          resolve((await vhd.readBlock(blockId)).data)
+        } catch (err) {
+          reject(err)
+        }
+      }
+      queueRunning = false
+    })()
+  }
+
+  async function readData(buf, len, pos) {
+    if (len === 0) return 0
+    const startBlockId = Math.floor(pos / blockSize)
+    // use (pos + len - 1) so an exact block boundary doesn't include the next block
+    const endBlockId = Math.floor((pos + len - 1) / blockSize)
+
+    let copied = 0
+    for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
+      const data = vhd.containsBlock(blockId) ? await enqueueBlock(blockId) : EMPTY_BLOCK
+      const offsetStart = blockId === startBlockId ? pos % blockSize : 0
+      const offsetEnd = blockId === endBlockId ? ((pos + len - 1) % blockSize) + 1 : blockSize
+      data.copy(buf, copied, offsetStart, offsetEnd)
+      copied += offsetEnd - offsetStart
+    }
+    return copied
+  }
+
   const fuse = new Fuse(mountDir, {
     async readdir(path, cb) {
-      if (path === '/') {
-        return cb(null, ['vhd0'])
-      }
+      if (path === '/') return cb(null, ['vhd0'])
       cb(Fuse.ENOENT)
     },
     async getattr(path, cb) {
-      if (path === '/') {
-        return cb(
-          null,
-          stat({
-            mode: 'dir',
-            size: 4096,
-          })
-        )
-      }
-      if (path === '/vhd0') {
-        return cb(
-          null,
-          stat({
-            mode: 'file',
-            size: vhd.footer.currentSize,
-          })
-        )
-      }
-
+      if (path === '/') return cb(null, stat({ mode: 'dir', size: 4096 }))
+      if (path === '/vhd0') return cb(null, stat({ mode: 'file', size: vhd.footer.currentSize }))
       cb(Fuse.ENOENT)
     },
     read(path, fd, buf, len, pos, cb) {
       if (path === '/vhd0') {
-        return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
+        return readData(buf, len, pos).then(cb, error => {
           warn('read error', { path, len, pos, error })
           cb(Fuse.EIO)
         })
@@ -64,7 +131,9 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
     },
   })
   return new Disposable(
-    () => fromCallback(cb => fuse.unmount(cb)),
+    () => {
+      return fromCallback(cb => fuse.unmount(cb))
+    },
     fromCallback(cb => fuse.mount(cb))
   )
 })

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -1,34 +1,23 @@
 import { asyncEach } from '@vates/async-each'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
-import { compose } from '@vates/compose'
 import { createLogger } from '@xen-orchestra/log'
 import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
 import { decorateMethodsWith } from '@vates/decorate-with'
-import { deduped } from '@vates/disposable/deduped.js'
 import { dirname, join, resolve } from 'node:path'
-import { execFile } from 'child_process'
-import { mount } from '@vates/fuse-vhd'
-import { readdir, lstat } from 'node:fs/promises'
 import { synchronized } from 'decorator-synchronized'
-import { ZipFile } from 'yazl'
 import Disposable from 'promise-toolbox/Disposable'
 import fromCallback from 'promise-toolbox/fromCallback'
-import fromEvent from 'promise-toolbox/fromEvent'
 import groupBy from 'lodash/groupBy.js'
-import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
 import reduce from 'lodash/reduce.js'
-import * as tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'
 import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
+import { fileRestoreDecorators, fileRestoreMethods } from './_fileRestore.mjs'
 import { formatFilenameDate } from './_filenameDate.mjs'
-import { getTmpDir } from './_getTmpDir.mjs'
 import { isMetadataFile } from './_backupType.mjs'
 import { isValidXva } from './_isValidXva.mjs'
-import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
-import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
 import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
@@ -48,25 +37,6 @@ export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 const noop = Function.prototype
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
-const makeRelative = path => resolve('/', path).slice(1)
-const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
-
-async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
-  for (const relativePath of relativePaths) {
-    const realPath = join(realBasePath, relativePath)
-    const virtualPath = join(virtualBasePath, relativePath)
-
-    const stats = await lstat(realPath)
-    const { mode, mtime } = stats
-    const opts = { mode, mtime }
-    if (stats.isDirectory()) {
-      zip.addEmptyDirectory(virtualPath, opts)
-      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
-    } else if (stats.isFile()) {
-      zip.addFile(realPath, virtualPath, opts)
-    }
-  }
-}
 
 const createSafeReaddir = (handler, methodName) => (path, options) =>
   handler.list(path, options).catch(error => {
@@ -76,11 +46,6 @@ const createSafeReaddir = (handler, methodName) => (path, options) =>
     return []
   })
 
-const debounceResourceFactory = factory =>
-  function () {
-    return this._debounceResource(factory.apply(this, arguments))
-  }
-
 export class RemoteAdapter {
   constructor(
     handler,
@@ -98,112 +63,6 @@ export class RemoteAdapter {
     return this._handler
   }
 
-  async _findPartition(devicePath, partitionId) {
-    const partitions = await listPartitions(devicePath)
-    const partition = partitions.find(_ => _.id === partitionId)
-    if (partition === undefined) {
-      throw new Error(`partition ${partitionId} not found`)
-    }
-    return partition
-  }
-
-  async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
-    yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
-
-    debug('activate LVM volume group', { vgName })
-    await fromCallback(execFile, 'vgchange', ['-ay', vgName])
-    try {
-      debug('get LVM volume group name and path', { vgName })
-      yield lvs(['lv_name', 'lv_path'], vgName)
-    } finally {
-      debug('deactivate LVM volume group', { vgName })
-      await fromCallback(execFile, 'vgchange', ['-an', vgName])
-    }
-  }
-
-  async *_getLvmPhysicalVolume(devicePath, partition) {
-    const args = []
-    if (partition !== undefined) {
-      args.push('-o', partition.start * 512, '--sizelimit', partition.size)
-    }
-    args.push('--show', '-f', devicePath)
-
-    debug('attach loop device', { devicePath, partition })
-    const path = (await fromCallback(execFile, 'losetup', args)).trim()
-    try {
-      debug('list LVM physical volume', { path })
-      await fromCallback(execFile, 'pvscan', ['--cache', path])
-
-      yield path
-    } finally {
-      try {
-        const vgNames = await pvs('vg_name', path)
-
-        debug('deactivate LVM volume groups', { vgNames })
-        await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
-      } finally {
-        debug('detach loop device', { path })
-        await fromCallback(execFile, 'losetup', ['-d', path])
-      }
-    }
-  }
-
-  async *_getPartition(devicePath, partition) {
-    // the norecovery option is necessary because if the partition is dirty,
-    // mount will try to fix it which is impossible if because the device is read-only
-    const options = ['loop', 'ro', 'norecovery']
-
-    if (partition !== undefined) {
-      const { size, start } = partition
-      options.push(`sizelimit=${size}`)
-      if (start !== undefined) {
-        options.push(`offset=${start * 512}`)
-      }
-    }
-
-    const path = yield getTmpDir()
-    const mount = options => {
-      debug('mount device', { devicePath, mountPath: path })
-      return fromCallback(execFile, 'mount', [
-        `--options=${options.join(',')}`,
-        `--source=${devicePath}`,
-        `--target=${path}`,
-      ])
-    }
-
-    // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
-    // another fs, try without
-    try {
-      await mount([...options, 'norecovery'])
-    } catch (error) {
-      await mount(options)
-    }
-    try {
-      yield path
-    } finally {
-      debug('umount device', { devicePath, mountPath: path })
-      await fromCallback(execFile, 'umount', ['--lazy', path])
-    }
-  }
-
-  _listLvmLogicalVolumes(devicePath, partition, results = []) {
-    return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => {
-      const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path)
-      const partitionId = partition !== undefined ? partition.id : ''
-      lvs.forEach((lv, i) => {
-        const name = lv.lv_name
-        if (name !== '') {
-          results.push({
-            id: `${partitionId}/${lv.vg_name}/${name}`,
-            name,
-            size: lv.lv_size,
-          })
-        }
-      })
-      return results
-    })
-  }
-
   // check if we will be allowed to merge a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
@@ -221,34 +80,6 @@ export class RemoteAdapter {
     })
   }
 
-  fetchPartitionFiles(diskId, partitionId, paths, format) {
-    const { promise, reject, resolve } = pDefer()
-    Disposable.use(
-      async function* () {
-        const path = yield this.getPartition(diskId, partitionId)
-        let outputStream
-
-        if (format === 'tgz') {
-          outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
-        } else if (format === 'zip') {
-          const zip = new ZipFile()
-          await addZipEntries(zip, path, '', paths.map(makeRelative))
-          zip.end()
-          ;({ outputStream } = zip)
-        } else {
-          throw new Error('unsupported format ' + format)
-        }
-
-        resolve(outputStream)
-        await fromEvent(outputStream, 'end')
-      }.bind(this)
-    ).catch(error => {
-      warn(error)
-      reject(error)
-    })
-    return promise
-  }
-
   async #removeVmBackupsFromCache(backups) {
     await asyncEach(
       Object.entries(
@@ -386,76 +217,6 @@ export class RemoteAdapter {
     return this.useVhdDirectory()
   }
 
-  async *#getDiskLegacy(diskId) {
-    const RE_VHDI = /^vhdi(\d+)$/
-    const handler = this._handler
-
-    const diskPath = handler.getFilePath('/' + diskId)
-    const mountDir = yield getTmpDir()
-
-    debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
-    await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
-    try {
-      let max = 0
-      let maxEntry
-      const entries = await readdir(mountDir)
-      entries.forEach(entry => {
-        const matches = RE_VHDI.exec(entry)
-        if (matches !== null) {
-          const value = +matches[1]
-          if (value > max) {
-            max = value
-            maxEntry = entry
-          }
-        }
-      })
-      if (max === 0) {
-        throw new Error('no disks found')
-      }
-
-      yield `${mountDir}/${maxEntry}`
-    } finally {
-      debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
-      await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
-    }
-  }
-
-  async *getDisk(diskId) {
-    if (this._useGetDiskLegacy) {
-      yield* this.#getDiskLegacy(diskId)
-      return
-    }
-    const handler = this._handler
-    // this is a disposable
-    const mountDir = yield getTmpDir()
-    // this is also a disposable
-    yield mount(handler, diskId, mountDir)
-    // this will yield disk path to caller
-    yield `${mountDir}/vhd0`
-  }
-
-  // partitionId values:
-  //
-  // - undefined: raw disk
-  // - `<partitionId>`: partitioned disk
-  // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
-  // - `/<vgName>/lvName>`: LVM on a raw disk
-  async *getPartition(diskId, partitionId) {
-    const devicePath = yield this.getDisk(diskId)
-    if (partitionId === undefined) {
-      return yield this._getPartition(devicePath)
-    }
-
-    const isLvmPartition = partitionId.includes('/')
-    if (isLvmPartition) {
-      const [pvId, vgName, lvName] = partitionId.split('/')
-      const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
-      return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
-    }
-
-    return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
-  }
-
   // if we use alias on this remote, we have to name the file alias.vhd
   getVhdFileName(baseName) {
     if (this.#useAlias()) {
@@ -496,56 +257,6 @@ export class RemoteAdapter {
     return backups
   }
 
-  listPartitionFiles(diskId, partitionId, path) {
-    return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
-      path = resolveSubpath(rootPath, path)
-      const entriesMap = {}
-      await asyncEach(
-        await readdir(path),
-        async name => {
-          try {
-            const stats = await lstat(`${path}/${name}`)
-            if (stats.isDirectory()) {
-              entriesMap[name + '/'] = {}
-            } else if (stats.isFile()) {
-              entriesMap[name] = {}
-            }
-          } catch (error) {
-            if (error == null || error.code !== 'ENOENT') {
-              throw error
-            }
-          }
-        },
-        { concurrency: 1 }
-      )
-
-      return entriesMap
-    })
-  }
-
-  listPartitions(diskId) {
-    return Disposable.use(this.getDisk(diskId), async devicePath => {
-      const partitions = await listPartitions(devicePath)
-
-      if (partitions.length === 0) {
-        try {
-          // handle potential raw LVM physical volume
-          return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
-        } catch (error) {
-          return []
-        }
-      }
-
-      const results = []
-      await asyncMapSettled(partitions, partition =>
-        partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
-          ? this._listLvmLogicalVolumes(devicePath, partition, results)
-          : results.push(partition)
-      )
-      return results
-    })
-  }
-
   async listPoolMetadataBackups() {
     const handler = this._handler
     const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
@@ -932,26 +643,8 @@ Object.assign(RemoteAdapter.prototype, {
   isValidXva,
 })
 
-decorateMethodsWith(RemoteAdapter, {
-  _getLvmLogicalVolumes: compose([
-    Disposable.factory,
-    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
-    debounceResourceFactory,
-  ]),
+// File-level-restore methods live in ./_fileRestore.mjs; mix them onto the prototype
+// before decorating so decorateMethodsWith can wrap them.
+Object.assign(RemoteAdapter.prototype, fileRestoreMethods)
 
-  _getLvmPhysicalVolume: compose([
-    Disposable.factory,
-    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
-    debounceResourceFactory,
-  ]),
-
-  _getPartition: compose([
-    Disposable.factory,
-    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
-    debounceResourceFactory,
-  ]),
-
-  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
-
-  getPartition: Disposable.factory,
-})
+decorateMethodsWith(RemoteAdapter, fileRestoreDecorators)

@xen-orchestra/backups/_fileRestore.integ.mjs

@@ -0,0 +1,94 @@
+import { strict as assert } from 'node:assert'
+import test from 'node:test'
+import { execFile } from 'node:child_process'
+import { copyFile, mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { randomBytes } from 'node:crypto'
+
+import Disposable from 'promise-toolbox/Disposable'
+
+import { RemoteAdapter } from './RemoteAdapter.mjs'
+
+// losetup/dmsetup/pvcreate/vgcreate/lvcreate/mkfs all require root (CAP_SYS_ADMIN).
+const skip =
+  typeof process.getuid !== 'function' || process.getuid() !== 0 ? 'requires root (losetup/dmsetup/lvm)' : false
+
+const pExec = promisify(execFile)
+const uniqueName = () => `xotest${randomBytes(6).toString('hex')}`
+
+async function withLoop(imagePath, fn) {
+  const loop = (await pExec('losetup', ['--show', '-f', imagePath])).stdout.trim()
+  try {
+    return await fn(loop)
+  } finally {
+    await pExec('losetup', ['-d', loop]).catch(() => {})
+  }
+}
+
+// Build a self-contained LVM image: a 64 MiB file holding one PV / VG / ext4 LV.
+async function createLvmImage(imagePath, vgName, lvName) {
+  await pExec('truncate', ['-s', '64M', imagePath])
+  await withLoop(imagePath, async loop => {
+    await pExec('pvcreate', ['-f', '-y', loop])
+    await pExec('vgcreate', [vgName, loop])
+    await pExec('lvcreate', ['-y', '-l', '100%FREE', '-n', lvName, vgName])
+    await pExec('mkfs.ext4', ['-q', `/dev/${vgName}/${lvName}`])
+    await pExec('vgchange', ['-an', vgName])
+  })
+}
+
+test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => {
+  const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-'))
+  const adapter = new RemoteAdapter({})
+  const vg = uniqueName()
+  const lv = 'root_lv'
+  const image = join(tmp, 'disk.raw')
+  const clone = join(tmp, 'clone.raw')
+  const plain = join(tmp, 'plain.raw')
+
+  t.after(async () => {
+    await rm(tmp, { recursive: true, force: true })
+    // drop any stale online-cache entries pointing at the now-deleted images
+    await pExec('pvscan', ['--cache']).catch(() => {})
+  })
+
+  await createLvmImage(image, vg, lv)
+
+  await t.test('detects the PV and renames the VG to a unique name', async () => {
+    await Disposable.use(adapter._getLvmPhysicalVolume(image, undefined), async ({ originalVgName, vgName }) => {
+      assert.equal(originalVgName, vg)
+      assert.match(vgName, /^xo[0-9a-f]{16}$/)
+      assert.notEqual(vgName, vg)
+    })
+  })
+
+  await t.test('two clones of the same VM (shared PVID) list concurrently without collision', async () => {
+    await copyFile(image, clone) // exact clone → identical PVID + VG name + VG UUID
+    const [a, b] = await Promise.all([
+      adapter._listLvmLogicalVolumes(image, undefined, []),
+      adapter._listLvmLogicalVolumes(clone, undefined, []),
+    ])
+    assert.equal(a.length, 1)
+    assert.equal(b.length, 1)
+    // the human-readable name is the (shared) original; the id carries the unique renamed VG
+    assert.equal(a[0].name, `${vg}/${lv}`)
+    assert.equal(b[0].name, `${vg}/${lv}`)
+    assert.notEqual(a[0].id, b[0].id)
+  })
+
+  await t.test('a non-LVM partition is rejected without building a snapshot', async () => {
+    await pExec('truncate', ['-s', '16M', plain])
+    await pExec('mkfs.ext4', ['-q', plain])
+    await assert.rejects(
+      Disposable.use(adapter._getLvmPhysicalVolume(plain, undefined), async () => {}),
+      /no LVM physical volume/
+    )
+  })
+
+  await t.test('leaves no dangling xo-pv-* dm-snapshots behind', async () => {
+    const { stdout } = await pExec('dmsetup', ['ls']).catch(() => ({ stdout: '' }))
+    assert.equal(/xo-pv-/.test(stdout), false, `leftover snapshots:\n${stdout}`)
+  })
+})

@xen-orchestra/backups/_fileRestore.mjs

@@ -0,0 +1,476 @@
+// File-level-restore (FLR) methods for RemoteAdapter.
+//
+// These were extracted from RemoteAdapter.mjs to keep that file focused on backup
+// CRUD/cache/metadata. They are mixed back onto `RemoteAdapter.prototype` via
+// `Object.assign` and decorated via `decorateMethodsWith` in RemoteAdapter.mjs, so `this`
+// is the RemoteAdapter instance at call time (`this._handler`, `this._debounceResource`,
+// `this._useGetDiskLegacy` keep working unchanged). The logger keeps the original
+// `xo:backups:RemoteAdapter` namespace so existing debug filters are unaffected.
+import { asyncEach } from '@vates/async-each'
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+import { compose } from '@vates/compose'
+import { createLogger } from '@xen-orchestra/log'
+import { deduped } from '@vates/disposable/deduped.js'
+import { randomBytes } from 'node:crypto'
+import { join, resolve } from 'node:path'
+import { execFile } from 'child_process'
+import { finished } from 'node:stream/promises'
+import { lstat, open, readdir, unlink } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { mount } from '@vates/fuse-vhd'
+import { ZipFile } from 'yazl'
+import Disposable from 'promise-toolbox/Disposable'
+import fromCallback from 'promise-toolbox/fromCallback'
+import pDefer from 'promise-toolbox/defer'
+import * as tar from 'tar'
+
+import { getTmpDir } from './_getTmpDir.mjs'
+import {
+  listPartitions,
+  LINUX_DATA_PARTITION_TYPE_GPT,
+  LINUX_DATA_PARTITION_TYPE_MBR,
+  LVM_PARTITION_TYPE_GPT,
+  LVM_PARTITION_TYPE_MBR,
+} from './_listPartitions.mjs'
+import { lvs, pvs } from './_lvm.mjs'
+
+const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
+
+const noop = Function.prototype
+
+// Restrict LVM scanning to a single device. Backup PVs are clones: the very same PVID
+// appears at once on the raw loop device, the dm-snapshot overlaid on it, and every
+// other restored copy of the same VM. An unscoped pvs/pvscan/vgimportclone/vgchange then
+// aborts with "duplicate PV ... for PVID ..." and the VG can neither be renamed nor
+// activated. Accepting only the device in hand removes every duplicate from LVM's view.
+export const lvmOnlyDevice = devicePath => `devices { global_filter=[ "a|^${devicePath}$|", "r|.*|" ] }`
+
+// Partition-type predicates (exported for unit tests).
+export const isLvmPartitionType = type => type === LVM_PARTITION_TYPE_MBR || type === LVM_PARTITION_TYPE_GPT
+
+// Some installers (e.g. Ubuntu subiquity) place an LVM PV on a generic Linux-data partition
+// instead of the LVM type, so those are the only non-LVM types worth the (expensive) probe.
+export const isProbeableForLvm = type =>
+  type === LINUX_DATA_PARTITION_TYPE_MBR || type === LINUX_DATA_PARTITION_TYPE_GPT
+
+// Build the partition entries for a PV's logical volumes (exported for unit tests).
+// Shows "ubuntu-vg/ubuntu-lv" for readability; skips unnamed LVs.
+export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
+  lvItems
+    .filter(lv => lv.lv_name !== '')
+    .map(lv => ({
+      id: `${partitionId}/${lv.vg_name}/${lv.lv_name}`,
+      name: originalVgName !== undefined ? `${originalVgName}/${lv.lv_name}` : lv.lv_name,
+      size: lv.lv_size,
+    }))
+
+const makeRelative = path => resolve('/', path).slice(1)
+const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
+
+async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
+  for (const relativePath of relativePaths) {
+    const realPath = join(realBasePath, relativePath)
+    const virtualPath = join(virtualBasePath, relativePath)
+
+    const stats = await lstat(realPath)
+    const { mode, mtime } = stats
+    const opts = { mode, mtime }
+    if (stats.isDirectory()) {
+      zip.addEmptyDirectory(virtualPath, opts)
+      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
+    } else if (stats.isFile()) {
+      zip.addFile(realPath, virtualPath, opts)
+    }
+  }
+}
+
+const debounceResourceFactory = factory =>
+  function () {
+    return this._debounceResource(factory.apply(this, arguments))
+  }
+
+// legacy disk mount via vhdimount; extracted from a private method so it can live in this
+// module (mixin methods cannot reference class-private members).
+async function* getDiskLegacy(handler, diskId) {
+  const RE_VHDI = /^vhdi(\d+)$/
+
+  const diskPath = handler.getFilePath('/' + diskId)
+  const mountDir = yield getTmpDir()
+
+  debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
+  await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
+  try {
+    let max = 0
+    let maxEntry
+    const entries = await readdir(mountDir)
+    entries.forEach(entry => {
+      const matches = RE_VHDI.exec(entry)
+      if (matches !== null) {
+        const value = +matches[1]
+        if (value > max) {
+          max = value
+          maxEntry = entry
+        }
+      }
+    })
+    if (max === 0) {
+      throw new Error('no disks found')
+    }
+
+    yield `${mountDir}/${maxEntry}`
+  } finally {
+    debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
+    await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
+  }
+}
+
+// Mixed onto RemoteAdapter.prototype — `this` is the RemoteAdapter instance.
+export const fileRestoreMethods = {
+  async _findPartition(devicePath, partitionId) {
+    const partitions = await listPartitions(devicePath)
+    const partition = partitions.find(_ => _.id === partitionId)
+    if (partition === undefined) {
+      throw new Error(`partition ${partitionId} not found`)
+    }
+    return partition
+  },
+
+  async *_getLvmLogicalVolumes(devicePath, pvId, requestedVgName) {
+    const { vgName, lvmConfig } = yield this._getLvmPhysicalVolume(
+      devicePath,
+      pvId && (await this._findPartition(devicePath, pvId))
+    )
+
+    // vgName is the unique name vgimportclone just assigned to this PV; prefer it over the
+    // (possibly stale) name embedded in the partitionId. lvmConfig scopes every command to
+    // this device so a colliding/duplicate VG on the host or another copy can't shadow it.
+    const effectiveVgName = vgName ?? requestedVgName
+
+    debug('activate LVM volume group', { effectiveVgName, requestedVgName })
+    await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-ay', effectiveVgName])
+    try {
+      debug('get LVM logical volumes', { effectiveVgName })
+      yield lvs(['lv_name', 'lv_path'], '--config', lvmConfig, effectiveVgName)
+    } finally {
+      debug('deactivate LVM volume group', { effectiveVgName })
+      await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', effectiveVgName])
+    }
+  },
+
+  async *_getLvmPhysicalVolume(devicePath, partition) {
+    const loopArgs = []
+    if (partition !== undefined) {
+      loopArgs.push('-o', partition.start * 512, '--sizelimit', partition.size)
+    }
+    loopArgs.push('--show', '-f', devicePath)
+
+    debug('attach loop device', { devicePath, partition })
+    const loopDevice = (await fromCallback(execFile, 'losetup', loopArgs)).trim()
+
+    let cowPath, cowLoop, mapperName, vgName, lvmConfig
+    try {
+      // Cheap pre-check, scoped to this loop so a duplicate PVID (another copy of the same
+      // VM restored concurrently) can't make it fail: is there an LVM PV here at all?
+      // Non-PV partitions (/boot, EFI, raw disks) skip the whole dm-snapshot machinery.
+      const [originalVgName] = (
+        await pvs('vg_name', '--config', lvmOnlyDevice(loopDevice), loopDevice).catch(() => [])
+      ).filter(Boolean)
+      if (originalVgName === undefined) {
+        const where = partition !== undefined ? ` partition ${partition.id}` : ''
+        throw new Error(`no LVM physical volume on ${devicePath}${where}`)
+      }
+
+      // The backup is read-only, so overlay a writable dm-snapshot to let vgimportclone
+      // rewrite metadata, and rename the VG to a unique name so concurrent clones (identical
+      // PVID and VG name) don't collide on device-mapper node names. Every LVM command is
+      // scoped to the snapshot device (lvmConfig) to keep duplicate PVIDs out of LVM's view.
+      mapperName = `xo-pv-${randomBytes(4).toString('hex')}`
+      const mapperPath = `/dev/mapper/${mapperName}`
+      lvmConfig = lvmOnlyDevice(mapperPath)
+
+      // ~4 MB sparse COW is enough to hold the LVM metadata rewrites
+      cowPath = join(tmpdir(), `${mapperName}.cow`)
+      const fh = await open(cowPath, 'w')
+      await fh.truncate(4 * 1024 * 1024)
+      await fh.close()
+      cowLoop = (await fromCallback(execFile, 'losetup', ['--show', '-f', cowPath])).trim()
+
+      const sectors = (await fromCallback(execFile, 'blockdev', ['--getsz', loopDevice])).trim()
+      await fromCallback(execFile, 'dmsetup', [
+        'create',
+        mapperName,
+        '--table',
+        `0 ${sectors} snapshot ${loopDevice} ${cowLoop} P 8`,
+      ])
+
+      // Unique random VG name: list and mount each query/yield the name from the device, so
+      // it need not be deterministic — only collision-free across concurrent clones.
+      vgName = `xo${randomBytes(8).toString('hex')}`
+      debug('import LVM volume group with unique name via dm-snapshot', { vgName, originalVgName })
+      await fromCallback(execFile, 'vgimportclone', ['--config', lvmConfig, '--basevgname', vgName, mapperPath])
+
+      yield { path: mapperPath, originalVgName, vgName, lvmConfig }
+    } finally {
+      if (mapperName !== undefined) {
+        // best-effort deactivate (a no-op if it was never activated) before removing the snapshot
+        await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', vgName]).catch(noop)
+        debug('remove dm-snapshot', { mapperName })
+        await fromCallback(execFile, 'dmsetup', ['remove', mapperName]).catch(err =>
+          warn('failed to remove dm-snapshot', { mapperName, error: err })
+        )
+      }
+      if (cowLoop !== undefined) {
+        await fromCallback(execFile, 'losetup', ['-d', cowLoop]).catch(noop)
+      }
+      if (cowPath !== undefined) {
+        await unlink(cowPath).catch(noop)
+      }
+      debug('detach loop device', { loopDevice })
+      await fromCallback(execFile, 'losetup', ['-d', loopDevice])
+    }
+  },
+
+  async *_getPartition(devicePath, partition) {
+    const options = ['loop', 'ro']
+
+    if (partition !== undefined) {
+      const { size, start } = partition
+      options.push(`sizelimit=${size}`)
+      if (start !== undefined) {
+        options.push(`offset=${start * 512}`)
+      }
+    }
+
+    const path = yield getTmpDir()
+    const mount = options => {
+      debug('mount device', { devicePath, mountPath: path })
+      return fromCallback(execFile, 'mount', [
+        `--options=${options.join(',')}`,
+        `--source=${devicePath}`,
+        `--target=${path}`,
+      ]).catch(error => {
+        if (error.stderr) {
+          error.message = `${error.message}: ${error.stderr.trim()}`
+        }
+        throw error
+      })
+    }
+
+    // norecovery prevents mount from attempting journal replay on a read-only device (ext3/ext4/xfs).
+    // Other filesystems don't support it, so fall back without it on failure.
+    try {
+      await mount([...options, 'norecovery'])
+    } catch (error) {
+      await mount(options)
+    }
+
+    try {
+      yield path
+    } finally {
+      debug('umount device', { devicePath, mountPath: path })
+      await fromCallback(execFile, 'umount', ['--lazy', path])
+    }
+  },
+
+  _listLvmLogicalVolumes(devicePath, partition, results = []) {
+    return Disposable.use(
+      this._getLvmPhysicalVolume(devicePath, partition),
+      async ({ path, originalVgName, lvmConfig }) => {
+        const lvItems = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], '--config', lvmConfig, path)
+        const partitionId = partition !== undefined ? partition.id : ''
+        results.push(...toLvPartitions(partitionId, originalVgName, lvItems))
+        return results
+      }
+    )
+  },
+
+  fetchPartitionFiles(diskId, partitionId, paths, format) {
+    const { promise, reject, resolve } = pDefer()
+    const self = this
+    Disposable.use(async function* () {
+      const path = yield self.getPartition(diskId, partitionId)
+      let outputStream
+
+      if (format === 'tgz') {
+        // process one entry at a time with { job: 1}. node-tar defaults to 4
+        // concurrent jobs, which on a FUSE-backed restore mount means up to 4
+        // simultaneous reads. Those saturate the libuv threadpool and starve
+        // the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
+        // threadpool deadlock). Serializing keeps a worker free for them.
+        outputStream = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
+        resolve(outputStream)
+      } else if (format === 'zip') {
+        const zip = new ZipFile()
+        // Resolve with the stream before enumeration so the client can start
+        // receiving data immediately — addZipEntries over FUSE/S3 can take
+        // minutes for large trees (e.g. node_modules) and would otherwise
+        // appear as a freeze with no response sent.
+        outputStream = zip.outputStream
+        resolve(zip.outputStream)
+        await addZipEntries(zip, path, '', paths.map(makeRelative))
+        zip.end()
+      } else {
+        throw new Error('unsupported format ' + format)
+      }
+
+      await finished(outputStream).catch(noop)
+    }).catch(error => {
+      warn(error)
+      reject(error)
+    })
+    return promise
+  },
+
+  async *getDisk(diskId) {
+    if (this._useGetDiskLegacy) {
+      yield* getDiskLegacy(this._handler, diskId)
+      return
+    }
+    const handler = this._handler
+    // this is a disposable
+    const mountDir = yield getTmpDir()
+    // this is also a disposable
+    yield mount(handler, diskId, mountDir)
+    // this will yield disk path to caller
+    yield `${mountDir}/vhd0`
+  },
+
+  // partitionId values:
+  //
+  // - undefined: raw disk
+  // - `<partitionId>`: partitioned disk
+  // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
+  // - `/<vgName>/lvName>`: LVM on a raw disk
+  async *getPartition(diskId, partitionId) {
+    const devicePath = yield this.getDisk(diskId)
+    if (partitionId === undefined) {
+      debug(
+        'no partition specified, attempting raw disk mount — call listPartitions first if disk has partitions or LVM'
+      )
+      return yield this._getPartition(devicePath)
+    }
+
+    const isLvmPartition = partitionId.includes('/')
+    if (isLvmPartition) {
+      const [pvId, vgName, lvName] = partitionId.split('/')
+      const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
+      return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
+    }
+
+    return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
+  },
+
+  listPartitionFiles(diskId, partitionId, path) {
+    return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
+      path = resolveSubpath(rootPath, path)
+      const entriesMap = {}
+      await asyncEach(
+        await readdir(path),
+        async name => {
+          try {
+            const stats = await lstat(`${path}/${name}`)
+            if (stats.isDirectory()) {
+              entriesMap[name + '/'] = {}
+            } else if (stats.isFile()) {
+              entriesMap[name] = {}
+            }
+          } catch (error) {
+            if (error == null || error.code !== 'ENOENT') {
+              throw error
+            }
+          }
+        },
+        { concurrency: 1 }
+      )
+
+      return entriesMap
+    })
+  },
+
+  listPartitions(diskId) {
+    return Disposable.use(this.getDisk(diskId), async devicePath => {
+      // partx may return empty on FUSE-backed files (vhd0); a loop device
+      // presents proper block-device semantics that partx reads reliably.
+      // losetup may itself fail if the FUSE mount isn't fully ready yet —
+      // fall back to the direct path so listPartitions returns [] instead of throwing.
+      let loopForParts, partitions
+      try {
+        loopForParts = (await fromCallback(execFile, 'losetup', ['--show', '-f', devicePath])).trim()
+        partitions = await listPartitions(loopForParts)
+      } catch (error) {
+        debug('partition probe via loop device failed, falling back to direct path', { error })
+        partitions = await listPartitions(devicePath)
+      } finally {
+        if (loopForParts !== undefined) {
+          await fromCallback(execFile, 'losetup', ['-d', loopForParts]).catch(noop)
+        }
+      }
+
+      if (partitions.length === 0) {
+        try {
+          // handle potential raw LVM physical volume
+          return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
+        } catch (error) {
+          return []
+        }
+      }
+
+      const results = []
+      await asyncMapSettled(partitions, async partition => {
+        if (isLvmPartitionType(partition.type)) {
+          return this._listLvmLogicalVolumes(devicePath, partition, results)
+        }
+
+        // Only generic Linux-data partitions can hide an LVM PV (subiquity-style); other
+        // types — BIOS boot, EFI, swap, … — are never PVs, so list them directly without
+        // the (expensive) loop + dm-snapshot + vgimportclone probe.
+        if (!isProbeableForLvm(partition.type)) {
+          results.push(partition)
+          return
+        }
+
+        const lvResults = []
+        try {
+          await this._listLvmLogicalVolumes(devicePath, partition, lvResults)
+        } catch (error) {
+          debug('LVM probe failed for Linux-data partition, treating as regular partition', {
+            partition,
+            error,
+          })
+        }
+        if (lvResults.length > 0) {
+          results.push(...lvResults)
+        } else {
+          results.push(partition)
+        }
+      })
+      return results
+    })
+  },
+}
+
+// Applied by RemoteAdapter.mjs via decorateMethodsWith(RemoteAdapter, fileRestoreDecorators).
+// debounceResourceFactory/deduped reference `this._debounceResource` at call time.
+export const fileRestoreDecorators = {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+}

@xen-orchestra/backups/_fileRestore.test.mjs

@@ -0,0 +1,61 @@
+import { strict as assert } from 'node:assert'
+import test from 'node:test'
+
+import { isLvmPartitionType, isProbeableForLvm, lvmOnlyDevice, toLvPartitions } from './_fileRestore.mjs'
+import {
+  LINUX_DATA_PARTITION_TYPE_GPT,
+  LINUX_DATA_PARTITION_TYPE_MBR,
+  LVM_PARTITION_TYPE_GPT,
+  LVM_PARTITION_TYPE_MBR,
+} from './_listPartitions.mjs'
+
+// well-known GPT type GUIDs that must NOT be probed/treated as LVM
+const EFI = 'c12a7328-f81f-11d2-ba4b-00a0c93ec93b'
+const BIOS_BOOT = '21686148-6449-6e6f-744e-656564454649'
+const SWAP = '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f'
+
+test('lvmOnlyDevice builds an accept-only global_filter for the device', () => {
+  assert.equal(
+    lvmOnlyDevice('/dev/mapper/xo-pv-abcd1234'),
+    'devices { global_filter=[ "a|^/dev/mapper/xo-pv-abcd1234$|", "r|.*|" ] }'
+  )
+  assert.equal(lvmOnlyDevice('/dev/loop7'), 'devices { global_filter=[ "a|^/dev/loop7$|", "r|.*|" ] }')
+})
+
+test('isLvmPartitionType matches only the LVM partition types', () => {
+  assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_MBR), true)
+  assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_GPT), true)
+  assert.equal(isLvmPartitionType(LINUX_DATA_PARTITION_TYPE_GPT), false)
+  assert.equal(isLvmPartitionType(EFI), false)
+  assert.equal(isLvmPartitionType(undefined), false)
+})
+
+test('isProbeableForLvm matches only generic Linux-data types', () => {
+  assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_MBR), true)
+  assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_GPT), true)
+  // never probe these
+  assert.equal(isProbeableForLvm(LVM_PARTITION_TYPE_GPT), false)
+  assert.equal(isProbeableForLvm(BIOS_BOOT), false)
+  assert.equal(isProbeableForLvm(EFI), false)
+  assert.equal(isProbeableForLvm(SWAP), false)
+  assert.equal(isProbeableForLvm(undefined), false)
+})
+
+test('toLvPartitions builds entries, skips unnamed LVs, shows the original VG name', () => {
+  const lvItems = [
+    { lv_name: 'ubuntu-lv', lv_path: '/dev/xohash/ubuntu-lv', lv_size: '1000', vg_name: 'xohash' },
+    // a PV with no LV yields a row with an empty lv_name → must be skipped
+    { lv_name: '', lv_path: '', lv_size: '0', vg_name: 'xohash' },
+    { lv_name: 'swap_1', lv_path: '/dev/xohash/swap_1', lv_size: '500', vg_name: 'xohash' },
+  ]
+  assert.deepEqual(toLvPartitions('part3', 'ubuntu-vg', lvItems), [
+    { id: 'part3/xohash/ubuntu-lv', name: 'ubuntu-vg/ubuntu-lv', size: '1000' },
+    { id: 'part3/xohash/swap_1', name: 'ubuntu-vg/swap_1', size: '500' },
+  ])
+})
+
+test('toLvPartitions falls back to the bare LV name when the original VG name is unknown', () => {
+  const lvItems = [{ lv_name: 'data', lv_path: '/dev/xohash/data', lv_size: '42', vg_name: 'xohash' }]
+  // raw-disk PV → empty partitionId
+  assert.deepEqual(toLvPartitions('', undefined, lvItems), [{ id: '/xohash/data', name: 'data', size: '42' }])
+})

@xen-orchestra/backups/_listPartitions.mjs

@@ -29,6 +29,15 @@ export const LVM_PARTITION_TYPE_MBR = 0x8e
 // GPT LVM type
 export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
 
+// Generic "Linux filesystem data" types. These are the only non-LVM-typed
+// partitions worth probing for a hidden LVM PV, because some installers (e.g.
+// Ubuntu subiquity) place a PV on a partition left with this generic type.
+// Other types (BIOS boot, EFI, swap, …) are never LVM PVs.
+// MBR Linux native
+export const LINUX_DATA_PARTITION_TYPE_MBR = 0x83
+// GPT Linux filesystem data
+export const LINUX_DATA_PARTITION_TYPE_GPT = '0fc63daf-8483-4772-8e79-3d69d8477de4'
+
 const parsePartxLine = createParser({
   keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
   valueTransform: (value, key) => {

@xen-orchestra/qa-test/package.json

@@ -43,6 +43,7 @@
     "qa:backup:metadata": "node --env-file-if-exists=.env --test tests/metadata-backup.test.js",
     "demo": "node --env-file-if-exists=.env index.js",
     "report": "node --env-file-if-exists=.env scripts/run-and-report.mjs",
-    "report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test"
+    "report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test",
+    "qa:backup:file-restore": "node --env-file-if-exists=.env --test tests/backup.file-restore.test.js"
   }
 }

@xen-orchestra/qa-test/tests/backup.file-restore.test.js

@@ -0,0 +1,181 @@
+import assert from 'node:assert'
+import { after, before, describe, it } from 'node:test'
+import { createLogger } from '@xen-orchestra/log'
+
+import { backupConfig } from '../backup.config.js'
+import { generateBackupJobName, getDefaultSchedule, getScheduleKey } from '../utils/index.js'
+import { assertBackupSuccess } from '../utils/backupUtils.js'
+import { setup, teardown } from './setup.js'
+
+const log = createLogger('qa:backup:file-restore')
+
+// Magic bytes used to validate archive format integrity
+const TGZ_MAGIC = [0x1f, 0x8b]
+const ZIP_MAGIC = [0x50, 0x4b]
+
+describe('Incremental backup file restore', () => {
+  let dispatchClient
+  let tracker
+  let vm
+  let backupRepository
+
+  // The second (delta) backup to run file restore on
+  let incrementalBackup
+
+  before(async () => {
+    ;({ dispatchClient, tracker, vm, backupRepository } = await setup())
+
+    const name = generateBackupJobName()
+    const schedule = getDefaultSchedule()
+    const config = backupConfig(name, schedule, vm, backupRepository)
+    config.mode = 'delta'
+
+    const backupJobId = await dispatchClient.backup.createBackupJob(config)
+    const backupJob = await dispatchClient.backup.details(backupJobId)
+
+    tracker.trackResource('backupJob', backupJobId, { name, mode: 'delta' })
+    const scheduleKey = getScheduleKey(backupJob)
+    if (scheduleKey) {
+      tracker.trackResource('schedule', scheduleKey, { name, backupJobId })
+    }
+
+    assert(scheduleKey, 'Schedule key is required for file restore tests')
+
+    // First run → full backup (base), second run → delta (incremental)
+    for (let i = 0; i < 2; i++) {
+      const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, scheduleKey)
+      assertBackupSuccess(result, `Backup run ${i + 1}/2`)
+      log.debug(`Completed backup run ${i + 1}/2`, { status: result.status })
+    }
+
+    // Pick the most recent delta backup for file restore testing
+    const backupsByRemoteAndVm = await dispatchClient.xoClient.call('backupNg.listVmBackups', {
+      remotes: [backupRepository.id],
+    })
+
+    const vmBackups = backupsByRemoteAndVm[backupRepository.id]?.[vm.uuid]
+    assert(Array.isArray(vmBackups) && vmBackups.length >= 2, 'Expected at least 2 backups after two runs')
+
+    const sorted = [...vmBackups].sort((a, b) => a.timestamp - b.timestamp)
+    incrementalBackup = sorted[sorted.length - 1]
+
+    assert.strictEqual(incrementalBackup.mode, 'delta', 'Latest backup should be a delta (incremental) backup')
+    assert(incrementalBackup.disks.length > 0, 'Incremental backup should contain at least one disk')
+
+    log.debug('Selected incremental backup for file restore', {
+      backupId: incrementalBackup.id,
+      diskCount: incrementalBackup.disks.length,
+      timestamp: new Date(incrementalBackup.timestamp).toISOString(),
+    })
+  })
+
+  after(async () => {
+    await teardown(dispatchClient, tracker)
+  })
+
+  it('should list partitions, list files, and restore first 2 files in tgz and zip for each disk', async () => {
+    const remoteId = backupRepository.id
+
+    for (const disk of incrementalBackup.disks) {
+      log.debug('Processing disk', { diskId: disk.id, diskName: disk.name })
+
+      const partitions = await dispatchClient.xoClient.call('backupNg.listPartitions', {
+        remote: remoteId,
+        disk: disk.id,
+      })
+
+      assert(Array.isArray(partitions), `listPartitions should return an array for disk ${disk.id}`)
+      log.debug('Found partitions', { diskId: disk.id, count: partitions.length })
+
+      // When listPartitions returns empty, treat the disk as a single raw (unpartitioned) volume.
+      // The API accepts partition: undefined for this case.
+      const partitionTargets = partitions.length > 0 ? partitions : [{ id: undefined }]
+
+      for (const partition of partitionTargets) {
+        const partitionId = partition.id
+        log.debug('Processing partition', { diskId: disk.id, partitionId })
+
+        let fileEntries
+        try {
+          fileEntries = await dispatchClient.xoClient.call('backupNg.listFiles', {
+            remote: remoteId,
+            disk: disk.id,
+            ...(partitionId !== undefined && { partition: partitionId }),
+            path: '/',
+          })
+        } catch (error) {
+          // Some partitions (swap, EFI system) cannot be mounted — skip gracefully
+          log.warn('Could not list files for partition, skipping', {
+            diskId: disk.id,
+            partitionId,
+            error: error.message,
+          })
+          continue
+        }
+
+        assert(
+          fileEntries !== null && typeof fileEntries === 'object',
+          `listFiles should return an object for disk ${disk.id} partition ${partitionId}`
+        )
+
+        // Directories are suffixed with '/', files are not
+        const fileNames = Object.keys(fileEntries).filter(name => !name.endsWith('/'))
+
+        if (fileNames.length === 0) {
+          log.debug('No files at partition root, skipping', { diskId: disk.id, partitionId })
+          continue
+        }
+
+        // Restore at most 2 files — paths must be absolute for fetchFiles
+        const paths = fileNames.slice(0, 2).map(name => `/${name}`)
+
+        log.debug('Restoring files', { diskId: disk.id, partitionId, paths })
+
+        for (const format of ['tgz', 'zip']) {
+          await assertFetchFiles(dispatchClient, { remote: remoteId, disk: disk.id, partitionId, paths, format })
+        }
+      }
+    }
+  })
+})
+
+/**
+ * Calls backupNg.fetchFiles, downloads the resulting archive, and validates
+ * that the response is a non-empty, correctly-formatted archive.
+ *
+ * @param {import('../client/dispatchClient.js').DispatchClient} dispatchClient
+ * @param {{ remote: string, disk: string, partitionId: string|undefined, paths: string[], format: 'tgz'|'zip' }} opts
+ */
+async function assertFetchFiles(dispatchClient, { remote, disk, partitionId, paths, format }) {
+  const params = {
+    remote,
+    disk,
+    paths,
+    format,
+    ...(partitionId !== undefined && { partition: partitionId }),
+  }
+
+  const result = await dispatchClient.xoClient.call('backupNg.fetchFiles', params)
+  const downloadPath = result.$getFrom
+  assert(
+    typeof downloadPath === 'string' && downloadPath.length > 0,
+    `fetchFiles (${format}) should return a non-empty $getFrom URL`
+  )
+
+  const response = await fetch(`${dispatchClient.restApiClient.baseUrl}${downloadPath}`, {
+    method: 'GET',
+    headers: dispatchClient.restApiClient.headers,
+    signal: AbortSignal.timeout(120_000),
+  })
+
+  assert(response.ok, `Download failed for ${format}: HTTP ${response.status} ${response.statusText}`)
+
+  const bytes = Buffer.from(await response.arrayBuffer())
+  assert(bytes.length > 0, `Downloaded ${format} archive should not be empty`)
+
+  const magic = format === 'tgz' ? TGZ_MAGIC : ZIP_MAGIC
+  assert.strictEqual(bytes[0], magic[0], `${format} archive should start with correct magic byte[0]`)
+  assert.strictEqual(bytes[1], magic[1], `${format} archive should start with correct magic byte[1]`)
+
+  log.debug('File restore validated', { format, size: bytes.length, paths, partitionId })
+}

CHANGELOG.unreleased.md

@@ -64,6 +64,8 @@
 - [Rest Api] Fix `possibly unhandled rejection invalid crendentials` (PR [#9938](https://github.com/vatesfr/xen-orchestra/pull/9938))
 - [Backups] Fixed "Cannot read properties of undefined" issues (PR [#9944](https://github.com/vatesfr/xen-orchestra/pull/9944))
 - [REST API] `GET /vms/:id.:format`, `GET /vm-templates/:id.:format`, `GET /vm-snapshots/:id.:format` now correctly support explicit compress query param (`zstd` | `gzip`). Still support `true` | `false` as deprecated value (PR [#9960](https://github.com/vatesfr/xen-orchestra/pull/9960))
+- [Backup/Restore] Fix file-level restore of VMs whose disks use LVM (e.g. the default Ubuntu install layout): logical volumes are now listed and can be restored, including when restoring several copies of the same VM at once — previously failed with `unknown filesystem type 'LVM2_member'` (PR [#9776](https://github.com/vatesfr/xen-orchestra/pull/9776))
+- [Backup/Restore] Fix file-level restore hanging when downloading large folders, and high memory use when downloading a folder as a zip (PR [#9776](https://github.com/vatesfr/xen-orchestra/pull/9776))
 
 ### Packages to release
 
@@ -81,11 +83,13 @@
 
 <!--packages-start-->
 
+- @vates/fuse-vhd patch
 - @vates/types minor
 - @xen-orchestra/acl minor
 - @xen-orchestra/backup-archive patch
+- @xen-orchestra/backups patch
 - @xen-orchestra/disk-transform patch
-- @xen-orchestra/qa-test patch
+- @xen-orchestra/qa-test minor
 - @xen-orchestra/qcow2 patch
 - @xen-orchestra/rest-api minor
 - @xen-orchestra/web minor

packages/xo-web/src/xo-app/backup/file-restore/restore-file-modal.js

@@ -18,9 +18,23 @@ import { listPartitions, listFiles } from 'xo'
 // -----------------------------------------------------------------------------
 
 const PARTITION_TYPE_NAMES = {
+  // MBR types
+  0x05: 'Extended',
   0x07: 'NTFS',
-  0x0c: 'FAT',
-  0x83: 'LINUX',
+  0x0b: 'FAT32',
+  0x0c: 'FAT32',
+  0x0e: 'FAT16',
+  0x82: 'Linux swap',
+  0x83: 'Linux',
+  0x8e: 'LVM',
+  0xfd: 'Linux RAID',
+  // GPT types
+  '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': 'Linux swap',
+  '0fc63daf-8483-4772-8e79-3d69d8477de4': 'Linux',
+  '21686148-6449-6e6f-744e-656564454649': 'BIOS boot',
+  'c12a7328-f81f-11d2-ba4b-00a0c93ec93b': 'EFI System',
+  'e6d6d379-f507-44c2-a23c-238f2a3df928': 'LVM',
+  'ebd0a0a2-b9e5-4433-87c0-68b6b72699c7': 'Windows',
 }
 
 const BACKUP_RENDERER = getRenderXoItemOfType('backup')
@@ -31,12 +45,16 @@ const diskOptionRenderer = disk => (
   </span>
 )
 
-const partitionOptionRenderer = partition => (
-  <span>
-    {partition.name} {defined(PARTITION_TYPE_NAMES[partition.type], partition.type)}{' '}
-    {partition.size && `(${formatSize(+partition.size)})`}
-  </span>
-)
+const partitionOptionRenderer = partition => {
+  const typeName = defined(PARTITION_TYPE_NAMES[partition.type], partition.type)
+  const label = [partition.name, typeName].filter(Boolean).join(' ')
+  return (
+    <span>
+      {label}
+      {partition.size && ` (${formatSize(+partition.size)})`}
+    </span>
+  )
+}
 
 const fileOptionRenderer = ({ isFile, name }) => (
   <span>