diff --git a/@vates/fuse-vhd/index.mjs b/@vates/fuse-vhd/index.mjs index 88f2694563..e5ddcff8a9 100644 --- a/@vates/fuse-vhd/index.mjs +++ b/@vates/fuse-vhd/index.mjs @@ -1,12 +1,12 @@ -import LRU from 'lru-cache' import Fuse from 'fuse-native' +import LRU from 'lru-cache' import { createLogger } from '@xen-orchestra/log' import { VhdSynthetic } from 'vhd-lib' import { Disposable, fromCallback } from 'promise-toolbox' const { warn } = createLogger('vates:fuse-vhd') -// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js +// build a stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js const stat = st => ({ mtime: st.mtime || new Date(), atime: st.atime || new Date(), @@ -20,42 +20,109 @@ const stat = st => ({ export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) { const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath) - const cache = new LRU({ - max: 16, // each cached block is 2MB in size - }) await vhd.readBlockAllocationTable() + const { blockSize } = vhd.header + + // A zeroed buffer returned for VHD blocks that are not present in the chain. + // Never mutated — safe to share across all reads. + const EMPTY_BLOCK = Buffer.alloc(blockSize, 0) + + // --- coalescing block read queue --- + // + // FUSE read requests from NTFS-3g arrive in small chunks (typically 4 KiB) + // while VHD blocks are 2 MiB. Reading blocks one at a time from the backing + // store (CIFS/NFS/…) bounds libuv threadpool usage to 1 concurrent I/O call. + // + // Coalescing: all pending requests for the same blockId share a single + // readBlock() call. When the block resolves, every waiter is fulfilled at once. + // + // Together these two properties prevent the FUSE-on-FUSE threadpool deadlock: + // concurrent NTFS-3g reads no longer starve the VHD block reads they depend on. + // In-memory block cache. Each VHD block is 2 MiB; 16 blocks = 32 MiB. + // This is the primary tool against the 300× CIFS amplification: NTFS-3g + // reads a 2 MiB block in ~512 sequential 4 KiB FUSE requests. Without a + // cache, each request causes a full 2 MiB CIFS read. With the cache, only + // the first request reads from CIFS; the rest are served from RAM. + const blockCache = new LRU({ max: 64 }) + + const pendingBlocks = new Map() // blockId → Promise + const readQueue = [] // { blockId, resolve, reject }[] + let queueRunning = false + + // log stats every 10 s while there is activity + + function enqueueBlock(blockId) { + // 1. serve from cache if available + if (blockCache.has(blockId)) { + return Promise.resolve(blockCache.get(blockId)) + } + // 2. coalesce: join an existing in-flight read for this block + if (pendingBlocks.has(blockId)) { + return pendingBlocks.get(blockId) + } + const p = new Promise((resolve, reject) => { + readQueue.push({ blockId, resolve, reject }) + }) + pendingBlocks.set(blockId, p) + p.then( + data => { + pendingBlocks.delete(blockId) + blockCache.set(blockId, data) + }, + () => pendingBlocks.delete(blockId) + ) + drainQueue() + return p + } + + function drainQueue() { + if (queueRunning) return + queueRunning = true + // async IIFE — errors are forwarded to individual reject() callbacks, + // so the outer floating promise is always fulfilled + ;(async () => { + while (readQueue.length > 0) { + const { blockId, resolve, reject } = readQueue.shift() + try { + resolve((await vhd.readBlock(blockId)).data) + } catch (err) { + reject(err) + } + } + queueRunning = false + })() + } + + async function readData(buf, len, pos) { + if (len === 0) return 0 + const startBlockId = Math.floor(pos / blockSize) + // use (pos + len - 1) so an exact block boundary doesn't include the next block + const endBlockId = Math.floor((pos + len - 1) / blockSize) + + let copied = 0 + for (let blockId = startBlockId; blockId <= endBlockId; blockId++) { + const data = vhd.containsBlock(blockId) ? await enqueueBlock(blockId) : EMPTY_BLOCK + const offsetStart = blockId === startBlockId ? pos % blockSize : 0 + const offsetEnd = blockId === endBlockId ? ((pos + len - 1) % blockSize) + 1 : blockSize + data.copy(buf, copied, offsetStart, offsetEnd) + copied += offsetEnd - offsetStart + } + return copied + } + const fuse = new Fuse(mountDir, { async readdir(path, cb) { - if (path === '/') { - return cb(null, ['vhd0']) - } + if (path === '/') return cb(null, ['vhd0']) cb(Fuse.ENOENT) }, async getattr(path, cb) { - if (path === '/') { - return cb( - null, - stat({ - mode: 'dir', - size: 4096, - }) - ) - } - if (path === '/vhd0') { - return cb( - null, - stat({ - mode: 'file', - size: vhd.footer.currentSize, - }) - ) - } - + if (path === '/') return cb(null, stat({ mode: 'dir', size: 4096 })) + if (path === '/vhd0') return cb(null, stat({ mode: 'file', size: vhd.footer.currentSize })) cb(Fuse.ENOENT) }, read(path, fd, buf, len, pos, cb) { if (path === '/vhd0') { - return vhd.readRawData(pos, len, cache, buf).then(cb, error => { + return readData(buf, len, pos).then(cb, error => { warn('read error', { path, len, pos, error }) cb(Fuse.EIO) }) @@ -64,7 +131,9 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath, }, }) return new Disposable( - () => fromCallback(cb => fuse.unmount(cb)), + () => { + return fromCallback(cb => fuse.unmount(cb)) + }, fromCallback(cb => fuse.mount(cb)) ) }) diff --git a/@xen-orchestra/backups/RemoteAdapter.mjs b/@xen-orchestra/backups/RemoteAdapter.mjs index 3403ac6b70..320acd4b35 100644 --- a/@xen-orchestra/backups/RemoteAdapter.mjs +++ b/@xen-orchestra/backups/RemoteAdapter.mjs @@ -1,34 +1,23 @@ import { asyncEach } from '@vates/async-each' import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map' -import { compose } from '@vates/compose' import { createLogger } from '@xen-orchestra/log' import { VhdDirectory, VhdSynthetic } from 'vhd-lib' import { decorateMethodsWith } from '@vates/decorate-with' -import { deduped } from '@vates/disposable/deduped.js' import { dirname, join, resolve } from 'node:path' -import { execFile } from 'child_process' -import { mount } from '@vates/fuse-vhd' -import { readdir, lstat } from 'node:fs/promises' import { synchronized } from 'decorator-synchronized' -import { ZipFile } from 'yazl' import Disposable from 'promise-toolbox/Disposable' import fromCallback from 'promise-toolbox/fromCallback' -import fromEvent from 'promise-toolbox/fromEvent' import groupBy from 'lodash/groupBy.js' -import pDefer from 'promise-toolbox/defer' import pickBy from 'lodash/pickBy.js' import reduce from 'lodash/reduce.js' -import * as tar from 'tar' import zlib from 'zlib' import { BACKUP_DIR } from './_getVmBackupDir.mjs' import { VmBackupDirectory } from '@xen-orchestra/backup-archive' +import { fileRestoreDecorators, fileRestoreMethods } from './_fileRestore.mjs' import { formatFilenameDate } from './_filenameDate.mjs' -import { getTmpDir } from './_getTmpDir.mjs' import { isMetadataFile } from './_backupType.mjs' import { isValidXva } from './_isValidXva.mjs' -import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs' -import { lvs, pvs } from './_lvm.mjs' import { watchStreamSize } from './_watchStreamSize.mjs' import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks' @@ -48,25 +37,6 @@ export const compareTimestamp = (a, b) => a.timestamp - b.timestamp const noop = Function.prototype const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1) -const makeRelative = path => resolve('/', path).slice(1) -const resolveSubpath = (root, path) => resolve(root, makeRelative(path)) - -async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) { - for (const relativePath of relativePaths) { - const realPath = join(realBasePath, relativePath) - const virtualPath = join(virtualBasePath, relativePath) - - const stats = await lstat(realPath) - const { mode, mtime } = stats - const opts = { mode, mtime } - if (stats.isDirectory()) { - zip.addEmptyDirectory(virtualPath, opts) - await addZipEntries(zip, realPath, virtualPath, await readdir(realPath)) - } else if (stats.isFile()) { - zip.addFile(realPath, virtualPath, opts) - } - } -} const createSafeReaddir = (handler, methodName) => (path, options) => handler.list(path, options).catch(error => { @@ -76,11 +46,6 @@ const createSafeReaddir = (handler, methodName) => (path, options) => return [] }) -const debounceResourceFactory = factory => - function () { - return this._debounceResource(factory.apply(this, arguments)) - } - export class RemoteAdapter { constructor( handler, @@ -98,112 +63,6 @@ export class RemoteAdapter { return this._handler } - async _findPartition(devicePath, partitionId) { - const partitions = await listPartitions(devicePath) - const partition = partitions.find(_ => _.id === partitionId) - if (partition === undefined) { - throw new Error(`partition ${partitionId} not found`) - } - return partition - } - - async *_getLvmLogicalVolumes(devicePath, pvId, vgName) { - yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId))) - - debug('activate LVM volume group', { vgName }) - await fromCallback(execFile, 'vgchange', ['-ay', vgName]) - try { - debug('get LVM volume group name and path', { vgName }) - yield lvs(['lv_name', 'lv_path'], vgName) - } finally { - debug('deactivate LVM volume group', { vgName }) - await fromCallback(execFile, 'vgchange', ['-an', vgName]) - } - } - - async *_getLvmPhysicalVolume(devicePath, partition) { - const args = [] - if (partition !== undefined) { - args.push('-o', partition.start * 512, '--sizelimit', partition.size) - } - args.push('--show', '-f', devicePath) - - debug('attach loop device', { devicePath, partition }) - const path = (await fromCallback(execFile, 'losetup', args)).trim() - try { - debug('list LVM physical volume', { path }) - await fromCallback(execFile, 'pvscan', ['--cache', path]) - - yield path - } finally { - try { - const vgNames = await pvs('vg_name', path) - - debug('deactivate LVM volume groups', { vgNames }) - await fromCallback(execFile, 'vgchange', ['-an', ...vgNames]) - } finally { - debug('detach loop device', { path }) - await fromCallback(execFile, 'losetup', ['-d', path]) - } - } - } - - async *_getPartition(devicePath, partition) { - // the norecovery option is necessary because if the partition is dirty, - // mount will try to fix it which is impossible if because the device is read-only - const options = ['loop', 'ro', 'norecovery'] - - if (partition !== undefined) { - const { size, start } = partition - options.push(`sizelimit=${size}`) - if (start !== undefined) { - options.push(`offset=${start * 512}`) - } - } - - const path = yield getTmpDir() - const mount = options => { - debug('mount device', { devicePath, mountPath: path }) - return fromCallback(execFile, 'mount', [ - `--options=${options.join(',')}`, - `--source=${devicePath}`, - `--target=${path}`, - ]) - } - - // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be - // another fs, try without - try { - await mount([...options, 'norecovery']) - } catch (error) { - await mount(options) - } - try { - yield path - } finally { - debug('umount device', { devicePath, mountPath: path }) - await fromCallback(execFile, 'umount', ['--lazy', path]) - } - } - - _listLvmLogicalVolumes(devicePath, partition, results = []) { - return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => { - const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path) - const partitionId = partition !== undefined ? partition.id : '' - lvs.forEach((lv, i) => { - const name = lv.lv_name - if (name !== '') { - results.push({ - id: `${partitionId}/${lv.vg_name}/${name}`, - name, - size: lv.lv_size, - }) - } - }) - return results - }) - } - // check if we will be allowed to merge a vhd created in this adapter // with the vhd at path `path` async isMergeableParent(packedParentUid, path) { @@ -221,34 +80,6 @@ export class RemoteAdapter { }) } - fetchPartitionFiles(diskId, partitionId, paths, format) { - const { promise, reject, resolve } = pDefer() - Disposable.use( - async function* () { - const path = yield this.getPartition(diskId, partitionId) - let outputStream - - if (format === 'tgz') { - outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative)) - } else if (format === 'zip') { - const zip = new ZipFile() - await addZipEntries(zip, path, '', paths.map(makeRelative)) - zip.end() - ;({ outputStream } = zip) - } else { - throw new Error('unsupported format ' + format) - } - - resolve(outputStream) - await fromEvent(outputStream, 'end') - }.bind(this) - ).catch(error => { - warn(error) - reject(error) - }) - return promise - } - async #removeVmBackupsFromCache(backups) { await asyncEach( Object.entries( @@ -386,76 +217,6 @@ export class RemoteAdapter { return this.useVhdDirectory() } - async *#getDiskLegacy(diskId) { - const RE_VHDI = /^vhdi(\d+)$/ - const handler = this._handler - - const diskPath = handler.getFilePath('/' + diskId) - const mountDir = yield getTmpDir() - - debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir }) - await fromCallback(execFile, 'vhdimount', [diskPath, mountDir]) - try { - let max = 0 - let maxEntry - const entries = await readdir(mountDir) - entries.forEach(entry => { - const matches = RE_VHDI.exec(entry) - if (matches !== null) { - const value = +matches[1] - if (value > max) { - max = value - maxEntry = entry - } - } - }) - if (max === 0) { - throw new Error('no disks found') - } - - yield `${mountDir}/${maxEntry}` - } finally { - debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir }) - await fromCallback(execFile, 'fusermount', ['-uz', mountDir]) - } - } - - async *getDisk(diskId) { - if (this._useGetDiskLegacy) { - yield* this.#getDiskLegacy(diskId) - return - } - const handler = this._handler - // this is a disposable - const mountDir = yield getTmpDir() - // this is also a disposable - yield mount(handler, diskId, mountDir) - // this will yield disk path to caller - yield `${mountDir}/vhd0` - } - - // partitionId values: - // - // - undefined: raw disk - // - ``: partitioned disk - // - `//`: LVM on a partitioned disk - // - `//lvName>`: LVM on a raw disk - async *getPartition(diskId, partitionId) { - const devicePath = yield this.getDisk(diskId) - if (partitionId === undefined) { - return yield this._getPartition(devicePath) - } - - const isLvmPartition = partitionId.includes('/') - if (isLvmPartition) { - const [pvId, vgName, lvName] = partitionId.split('/') - const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName) - return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path) - } - - return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId)) - } - // if we use alias on this remote, we have to name the file alias.vhd getVhdFileName(baseName) { if (this.#useAlias()) { @@ -496,56 +257,6 @@ export class RemoteAdapter { return backups } - listPartitionFiles(diskId, partitionId, path) { - return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => { - path = resolveSubpath(rootPath, path) - const entriesMap = {} - await asyncEach( - await readdir(path), - async name => { - try { - const stats = await lstat(`${path}/${name}`) - if (stats.isDirectory()) { - entriesMap[name + '/'] = {} - } else if (stats.isFile()) { - entriesMap[name] = {} - } - } catch (error) { - if (error == null || error.code !== 'ENOENT') { - throw error - } - } - }, - { concurrency: 1 } - ) - - return entriesMap - }) - } - - listPartitions(diskId) { - return Disposable.use(this.getDisk(diskId), async devicePath => { - const partitions = await listPartitions(devicePath) - - if (partitions.length === 0) { - try { - // handle potential raw LVM physical volume - return await this._listLvmLogicalVolumes(devicePath, undefined, partitions) - } catch (error) { - return [] - } - } - - const results = [] - await asyncMapSettled(partitions, partition => - partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT - ? this._listLvmLogicalVolumes(devicePath, partition, results) - : results.push(partition) - ) - return results - }) - } - async listPoolMetadataBackups() { const handler = this._handler const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups') @@ -932,26 +643,8 @@ Object.assign(RemoteAdapter.prototype, { isValidXva, }) -decorateMethodsWith(RemoteAdapter, { - _getLvmLogicalVolumes: compose([ - Disposable.factory, - [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]], - debounceResourceFactory, - ]), +// File-level-restore methods live in ./_fileRestore.mjs; mix them onto the prototype +// before decorating so decorateMethodsWith can wrap them. +Object.assign(RemoteAdapter.prototype, fileRestoreMethods) - _getLvmPhysicalVolume: compose([ - Disposable.factory, - [deduped, (devicePath, partition) => [devicePath, partition?.id]], - debounceResourceFactory, - ]), - - _getPartition: compose([ - Disposable.factory, - [deduped, (devicePath, partition) => [devicePath, partition?.id]], - debounceResourceFactory, - ]), - - getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]), - - getPartition: Disposable.factory, -}) +decorateMethodsWith(RemoteAdapter, fileRestoreDecorators) diff --git a/@xen-orchestra/backups/_fileRestore.integ.mjs b/@xen-orchestra/backups/_fileRestore.integ.mjs new file mode 100644 index 0000000000..7d390e57d4 --- /dev/null +++ b/@xen-orchestra/backups/_fileRestore.integ.mjs @@ -0,0 +1,94 @@ +import { strict as assert } from 'node:assert' +import test from 'node:test' +import { execFile } from 'node:child_process' +import { copyFile, mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { randomBytes } from 'node:crypto' + +import Disposable from 'promise-toolbox/Disposable' + +import { RemoteAdapter } from './RemoteAdapter.mjs' + +// losetup/dmsetup/pvcreate/vgcreate/lvcreate/mkfs all require root (CAP_SYS_ADMIN). +const skip = + typeof process.getuid !== 'function' || process.getuid() !== 0 ? 'requires root (losetup/dmsetup/lvm)' : false + +const pExec = promisify(execFile) +const uniqueName = () => `xotest${randomBytes(6).toString('hex')}` + +async function withLoop(imagePath, fn) { + const loop = (await pExec('losetup', ['--show', '-f', imagePath])).stdout.trim() + try { + return await fn(loop) + } finally { + await pExec('losetup', ['-d', loop]).catch(() => {}) + } +} + +// Build a self-contained LVM image: a 64 MiB file holding one PV / VG / ext4 LV. +async function createLvmImage(imagePath, vgName, lvName) { + await pExec('truncate', ['-s', '64M', imagePath]) + await withLoop(imagePath, async loop => { + await pExec('pvcreate', ['-f', '-y', loop]) + await pExec('vgcreate', [vgName, loop]) + await pExec('lvcreate', ['-y', '-l', '100%FREE', '-n', lvName, vgName]) + await pExec('mkfs.ext4', ['-q', `/dev/${vgName}/${lvName}`]) + await pExec('vgchange', ['-an', vgName]) + }) +} + +test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => { + const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-')) + const adapter = new RemoteAdapter({}) + const vg = uniqueName() + const lv = 'root_lv' + const image = join(tmp, 'disk.raw') + const clone = join(tmp, 'clone.raw') + const plain = join(tmp, 'plain.raw') + + t.after(async () => { + await rm(tmp, { recursive: true, force: true }) + // drop any stale online-cache entries pointing at the now-deleted images + await pExec('pvscan', ['--cache']).catch(() => {}) + }) + + await createLvmImage(image, vg, lv) + + await t.test('detects the PV and renames the VG to a unique name', async () => { + await Disposable.use(adapter._getLvmPhysicalVolume(image, undefined), async ({ originalVgName, vgName }) => { + assert.equal(originalVgName, vg) + assert.match(vgName, /^xo[0-9a-f]{16}$/) + assert.notEqual(vgName, vg) + }) + }) + + await t.test('two clones of the same VM (shared PVID) list concurrently without collision', async () => { + await copyFile(image, clone) // exact clone → identical PVID + VG name + VG UUID + const [a, b] = await Promise.all([ + adapter._listLvmLogicalVolumes(image, undefined, []), + adapter._listLvmLogicalVolumes(clone, undefined, []), + ]) + assert.equal(a.length, 1) + assert.equal(b.length, 1) + // the human-readable name is the (shared) original; the id carries the unique renamed VG + assert.equal(a[0].name, `${vg}/${lv}`) + assert.equal(b[0].name, `${vg}/${lv}`) + assert.notEqual(a[0].id, b[0].id) + }) + + await t.test('a non-LVM partition is rejected without building a snapshot', async () => { + await pExec('truncate', ['-s', '16M', plain]) + await pExec('mkfs.ext4', ['-q', plain]) + await assert.rejects( + Disposable.use(adapter._getLvmPhysicalVolume(plain, undefined), async () => {}), + /no LVM physical volume/ + ) + }) + + await t.test('leaves no dangling xo-pv-* dm-snapshots behind', async () => { + const { stdout } = await pExec('dmsetup', ['ls']).catch(() => ({ stdout: '' })) + assert.equal(/xo-pv-/.test(stdout), false, `leftover snapshots:\n${stdout}`) + }) +}) diff --git a/@xen-orchestra/backups/_fileRestore.mjs b/@xen-orchestra/backups/_fileRestore.mjs new file mode 100644 index 0000000000..0002e099a3 --- /dev/null +++ b/@xen-orchestra/backups/_fileRestore.mjs @@ -0,0 +1,476 @@ +// File-level-restore (FLR) methods for RemoteAdapter. +// +// These were extracted from RemoteAdapter.mjs to keep that file focused on backup +// CRUD/cache/metadata. They are mixed back onto `RemoteAdapter.prototype` via +// `Object.assign` and decorated via `decorateMethodsWith` in RemoteAdapter.mjs, so `this` +// is the RemoteAdapter instance at call time (`this._handler`, `this._debounceResource`, +// `this._useGetDiskLegacy` keep working unchanged). The logger keeps the original +// `xo:backups:RemoteAdapter` namespace so existing debug filters are unaffected. +import { asyncEach } from '@vates/async-each' +import { asyncMapSettled } from '@xen-orchestra/async-map' +import { compose } from '@vates/compose' +import { createLogger } from '@xen-orchestra/log' +import { deduped } from '@vates/disposable/deduped.js' +import { randomBytes } from 'node:crypto' +import { join, resolve } from 'node:path' +import { execFile } from 'child_process' +import { finished } from 'node:stream/promises' +import { lstat, open, readdir, unlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { mount } from '@vates/fuse-vhd' +import { ZipFile } from 'yazl' +import Disposable from 'promise-toolbox/Disposable' +import fromCallback from 'promise-toolbox/fromCallback' +import pDefer from 'promise-toolbox/defer' +import * as tar from 'tar' + +import { getTmpDir } from './_getTmpDir.mjs' +import { + listPartitions, + LINUX_DATA_PARTITION_TYPE_GPT, + LINUX_DATA_PARTITION_TYPE_MBR, + LVM_PARTITION_TYPE_GPT, + LVM_PARTITION_TYPE_MBR, +} from './_listPartitions.mjs' +import { lvs, pvs } from './_lvm.mjs' + +const { debug, warn } = createLogger('xo:backups:RemoteAdapter') + +const noop = Function.prototype + +// Restrict LVM scanning to a single device. Backup PVs are clones: the very same PVID +// appears at once on the raw loop device, the dm-snapshot overlaid on it, and every +// other restored copy of the same VM. An unscoped pvs/pvscan/vgimportclone/vgchange then +// aborts with "duplicate PV ... for PVID ..." and the VG can neither be renamed nor +// activated. Accepting only the device in hand removes every duplicate from LVM's view. +export const lvmOnlyDevice = devicePath => `devices { global_filter=[ "a|^${devicePath}$|", "r|.*|" ] }` + +// Partition-type predicates (exported for unit tests). +export const isLvmPartitionType = type => type === LVM_PARTITION_TYPE_MBR || type === LVM_PARTITION_TYPE_GPT + +// Some installers (e.g. Ubuntu subiquity) place an LVM PV on a generic Linux-data partition +// instead of the LVM type, so those are the only non-LVM types worth the (expensive) probe. +export const isProbeableForLvm = type => + type === LINUX_DATA_PARTITION_TYPE_MBR || type === LINUX_DATA_PARTITION_TYPE_GPT + +// Build the partition entries for a PV's logical volumes (exported for unit tests). +// Shows "ubuntu-vg/ubuntu-lv" for readability; skips unnamed LVs. +export const toLvPartitions = (partitionId, originalVgName, lvItems) => + lvItems + .filter(lv => lv.lv_name !== '') + .map(lv => ({ + id: `${partitionId}/${lv.vg_name}/${lv.lv_name}`, + name: originalVgName !== undefined ? `${originalVgName}/${lv.lv_name}` : lv.lv_name, + size: lv.lv_size, + })) + +const makeRelative = path => resolve('/', path).slice(1) +const resolveSubpath = (root, path) => resolve(root, makeRelative(path)) + +async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) { + for (const relativePath of relativePaths) { + const realPath = join(realBasePath, relativePath) + const virtualPath = join(virtualBasePath, relativePath) + + const stats = await lstat(realPath) + const { mode, mtime } = stats + const opts = { mode, mtime } + if (stats.isDirectory()) { + zip.addEmptyDirectory(virtualPath, opts) + await addZipEntries(zip, realPath, virtualPath, await readdir(realPath)) + } else if (stats.isFile()) { + zip.addFile(realPath, virtualPath, opts) + } + } +} + +const debounceResourceFactory = factory => + function () { + return this._debounceResource(factory.apply(this, arguments)) + } + +// legacy disk mount via vhdimount; extracted from a private method so it can live in this +// module (mixin methods cannot reference class-private members). +async function* getDiskLegacy(handler, diskId) { + const RE_VHDI = /^vhdi(\d+)$/ + + const diskPath = handler.getFilePath('/' + diskId) + const mountDir = yield getTmpDir() + + debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir }) + await fromCallback(execFile, 'vhdimount', [diskPath, mountDir]) + try { + let max = 0 + let maxEntry + const entries = await readdir(mountDir) + entries.forEach(entry => { + const matches = RE_VHDI.exec(entry) + if (matches !== null) { + const value = +matches[1] + if (value > max) { + max = value + maxEntry = entry + } + } + }) + if (max === 0) { + throw new Error('no disks found') + } + + yield `${mountDir}/${maxEntry}` + } finally { + debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir }) + await fromCallback(execFile, 'fusermount', ['-uz', mountDir]) + } +} + +// Mixed onto RemoteAdapter.prototype — `this` is the RemoteAdapter instance. +export const fileRestoreMethods = { + async _findPartition(devicePath, partitionId) { + const partitions = await listPartitions(devicePath) + const partition = partitions.find(_ => _.id === partitionId) + if (partition === undefined) { + throw new Error(`partition ${partitionId} not found`) + } + return partition + }, + + async *_getLvmLogicalVolumes(devicePath, pvId, requestedVgName) { + const { vgName, lvmConfig } = yield this._getLvmPhysicalVolume( + devicePath, + pvId && (await this._findPartition(devicePath, pvId)) + ) + + // vgName is the unique name vgimportclone just assigned to this PV; prefer it over the + // (possibly stale) name embedded in the partitionId. lvmConfig scopes every command to + // this device so a colliding/duplicate VG on the host or another copy can't shadow it. + const effectiveVgName = vgName ?? requestedVgName + + debug('activate LVM volume group', { effectiveVgName, requestedVgName }) + await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-ay', effectiveVgName]) + try { + debug('get LVM logical volumes', { effectiveVgName }) + yield lvs(['lv_name', 'lv_path'], '--config', lvmConfig, effectiveVgName) + } finally { + debug('deactivate LVM volume group', { effectiveVgName }) + await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', effectiveVgName]) + } + }, + + async *_getLvmPhysicalVolume(devicePath, partition) { + const loopArgs = [] + if (partition !== undefined) { + loopArgs.push('-o', partition.start * 512, '--sizelimit', partition.size) + } + loopArgs.push('--show', '-f', devicePath) + + debug('attach loop device', { devicePath, partition }) + const loopDevice = (await fromCallback(execFile, 'losetup', loopArgs)).trim() + + let cowPath, cowLoop, mapperName, vgName, lvmConfig + try { + // Cheap pre-check, scoped to this loop so a duplicate PVID (another copy of the same + // VM restored concurrently) can't make it fail: is there an LVM PV here at all? + // Non-PV partitions (/boot, EFI, raw disks) skip the whole dm-snapshot machinery. + const [originalVgName] = ( + await pvs('vg_name', '--config', lvmOnlyDevice(loopDevice), loopDevice).catch(() => []) + ).filter(Boolean) + if (originalVgName === undefined) { + const where = partition !== undefined ? ` partition ${partition.id}` : '' + throw new Error(`no LVM physical volume on ${devicePath}${where}`) + } + + // The backup is read-only, so overlay a writable dm-snapshot to let vgimportclone + // rewrite metadata, and rename the VG to a unique name so concurrent clones (identical + // PVID and VG name) don't collide on device-mapper node names. Every LVM command is + // scoped to the snapshot device (lvmConfig) to keep duplicate PVIDs out of LVM's view. + mapperName = `xo-pv-${randomBytes(4).toString('hex')}` + const mapperPath = `/dev/mapper/${mapperName}` + lvmConfig = lvmOnlyDevice(mapperPath) + + // ~4 MB sparse COW is enough to hold the LVM metadata rewrites + cowPath = join(tmpdir(), `${mapperName}.cow`) + const fh = await open(cowPath, 'w') + await fh.truncate(4 * 1024 * 1024) + await fh.close() + cowLoop = (await fromCallback(execFile, 'losetup', ['--show', '-f', cowPath])).trim() + + const sectors = (await fromCallback(execFile, 'blockdev', ['--getsz', loopDevice])).trim() + await fromCallback(execFile, 'dmsetup', [ + 'create', + mapperName, + '--table', + `0 ${sectors} snapshot ${loopDevice} ${cowLoop} P 8`, + ]) + + // Unique random VG name: list and mount each query/yield the name from the device, so + // it need not be deterministic — only collision-free across concurrent clones. + vgName = `xo${randomBytes(8).toString('hex')}` + debug('import LVM volume group with unique name via dm-snapshot', { vgName, originalVgName }) + await fromCallback(execFile, 'vgimportclone', ['--config', lvmConfig, '--basevgname', vgName, mapperPath]) + + yield { path: mapperPath, originalVgName, vgName, lvmConfig } + } finally { + if (mapperName !== undefined) { + // best-effort deactivate (a no-op if it was never activated) before removing the snapshot + await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', vgName]).catch(noop) + debug('remove dm-snapshot', { mapperName }) + await fromCallback(execFile, 'dmsetup', ['remove', mapperName]).catch(err => + warn('failed to remove dm-snapshot', { mapperName, error: err }) + ) + } + if (cowLoop !== undefined) { + await fromCallback(execFile, 'losetup', ['-d', cowLoop]).catch(noop) + } + if (cowPath !== undefined) { + await unlink(cowPath).catch(noop) + } + debug('detach loop device', { loopDevice }) + await fromCallback(execFile, 'losetup', ['-d', loopDevice]) + } + }, + + async *_getPartition(devicePath, partition) { + const options = ['loop', 'ro'] + + if (partition !== undefined) { + const { size, start } = partition + options.push(`sizelimit=${size}`) + if (start !== undefined) { + options.push(`offset=${start * 512}`) + } + } + + const path = yield getTmpDir() + const mount = options => { + debug('mount device', { devicePath, mountPath: path }) + return fromCallback(execFile, 'mount', [ + `--options=${options.join(',')}`, + `--source=${devicePath}`, + `--target=${path}`, + ]).catch(error => { + if (error.stderr) { + error.message = `${error.message}: ${error.stderr.trim()}` + } + throw error + }) + } + + // norecovery prevents mount from attempting journal replay on a read-only device (ext3/ext4/xfs). + // Other filesystems don't support it, so fall back without it on failure. + try { + await mount([...options, 'norecovery']) + } catch (error) { + await mount(options) + } + + try { + yield path + } finally { + debug('umount device', { devicePath, mountPath: path }) + await fromCallback(execFile, 'umount', ['--lazy', path]) + } + }, + + _listLvmLogicalVolumes(devicePath, partition, results = []) { + return Disposable.use( + this._getLvmPhysicalVolume(devicePath, partition), + async ({ path, originalVgName, lvmConfig }) => { + const lvItems = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], '--config', lvmConfig, path) + const partitionId = partition !== undefined ? partition.id : '' + results.push(...toLvPartitions(partitionId, originalVgName, lvItems)) + return results + } + ) + }, + + fetchPartitionFiles(diskId, partitionId, paths, format) { + const { promise, reject, resolve } = pDefer() + const self = this + Disposable.use(async function* () { + const path = yield self.getPartition(diskId, partitionId) + let outputStream + + if (format === 'tgz') { + // process one entry at a time with { job: 1}. node-tar defaults to 4 + // concurrent jobs, which on a FUSE-backed restore mount means up to 4 + // simultaneous reads. Those saturate the libuv threadpool and starve + // the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE + // threadpool deadlock). Serializing keeps a worker free for them. + outputStream = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative)) + resolve(outputStream) + } else if (format === 'zip') { + const zip = new ZipFile() + // Resolve with the stream before enumeration so the client can start + // receiving data immediately — addZipEntries over FUSE/S3 can take + // minutes for large trees (e.g. node_modules) and would otherwise + // appear as a freeze with no response sent. + outputStream = zip.outputStream + resolve(zip.outputStream) + await addZipEntries(zip, path, '', paths.map(makeRelative)) + zip.end() + } else { + throw new Error('unsupported format ' + format) + } + + await finished(outputStream).catch(noop) + }).catch(error => { + warn(error) + reject(error) + }) + return promise + }, + + async *getDisk(diskId) { + if (this._useGetDiskLegacy) { + yield* getDiskLegacy(this._handler, diskId) + return + } + const handler = this._handler + // this is a disposable + const mountDir = yield getTmpDir() + // this is also a disposable + yield mount(handler, diskId, mountDir) + // this will yield disk path to caller + yield `${mountDir}/vhd0` + }, + + // partitionId values: + // + // - undefined: raw disk + // - ``: partitioned disk + // - `//`: LVM on a partitioned disk + // - `//lvName>`: LVM on a raw disk + async *getPartition(diskId, partitionId) { + const devicePath = yield this.getDisk(diskId) + if (partitionId === undefined) { + debug( + 'no partition specified, attempting raw disk mount — call listPartitions first if disk has partitions or LVM' + ) + return yield this._getPartition(devicePath) + } + + const isLvmPartition = partitionId.includes('/') + if (isLvmPartition) { + const [pvId, vgName, lvName] = partitionId.split('/') + const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName) + return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path) + } + + return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId)) + }, + + listPartitionFiles(diskId, partitionId, path) { + return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => { + path = resolveSubpath(rootPath, path) + const entriesMap = {} + await asyncEach( + await readdir(path), + async name => { + try { + const stats = await lstat(`${path}/${name}`) + if (stats.isDirectory()) { + entriesMap[name + '/'] = {} + } else if (stats.isFile()) { + entriesMap[name] = {} + } + } catch (error) { + if (error == null || error.code !== 'ENOENT') { + throw error + } + } + }, + { concurrency: 1 } + ) + + return entriesMap + }) + }, + + listPartitions(diskId) { + return Disposable.use(this.getDisk(diskId), async devicePath => { + // partx may return empty on FUSE-backed files (vhd0); a loop device + // presents proper block-device semantics that partx reads reliably. + // losetup may itself fail if the FUSE mount isn't fully ready yet — + // fall back to the direct path so listPartitions returns [] instead of throwing. + let loopForParts, partitions + try { + loopForParts = (await fromCallback(execFile, 'losetup', ['--show', '-f', devicePath])).trim() + partitions = await listPartitions(loopForParts) + } catch (error) { + debug('partition probe via loop device failed, falling back to direct path', { error }) + partitions = await listPartitions(devicePath) + } finally { + if (loopForParts !== undefined) { + await fromCallback(execFile, 'losetup', ['-d', loopForParts]).catch(noop) + } + } + + if (partitions.length === 0) { + try { + // handle potential raw LVM physical volume + return await this._listLvmLogicalVolumes(devicePath, undefined, partitions) + } catch (error) { + return [] + } + } + + const results = [] + await asyncMapSettled(partitions, async partition => { + if (isLvmPartitionType(partition.type)) { + return this._listLvmLogicalVolumes(devicePath, partition, results) + } + + // Only generic Linux-data partitions can hide an LVM PV (subiquity-style); other + // types — BIOS boot, EFI, swap, … — are never PVs, so list them directly without + // the (expensive) loop + dm-snapshot + vgimportclone probe. + if (!isProbeableForLvm(partition.type)) { + results.push(partition) + return + } + + const lvResults = [] + try { + await this._listLvmLogicalVolumes(devicePath, partition, lvResults) + } catch (error) { + debug('LVM probe failed for Linux-data partition, treating as regular partition', { + partition, + error, + }) + } + if (lvResults.length > 0) { + results.push(...lvResults) + } else { + results.push(partition) + } + }) + return results + }) + }, +} + +// Applied by RemoteAdapter.mjs via decorateMethodsWith(RemoteAdapter, fileRestoreDecorators). +// debounceResourceFactory/deduped reference `this._debounceResource` at call time. +export const fileRestoreDecorators = { + _getLvmLogicalVolumes: compose([ + Disposable.factory, + [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]], + debounceResourceFactory, + ]), + + _getLvmPhysicalVolume: compose([ + Disposable.factory, + [deduped, (devicePath, partition) => [devicePath, partition?.id]], + debounceResourceFactory, + ]), + + _getPartition: compose([ + Disposable.factory, + [deduped, (devicePath, partition) => [devicePath, partition?.id]], + debounceResourceFactory, + ]), + + getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]), + + getPartition: Disposable.factory, +} diff --git a/@xen-orchestra/backups/_fileRestore.test.mjs b/@xen-orchestra/backups/_fileRestore.test.mjs new file mode 100644 index 0000000000..4a349acb66 --- /dev/null +++ b/@xen-orchestra/backups/_fileRestore.test.mjs @@ -0,0 +1,61 @@ +import { strict as assert } from 'node:assert' +import test from 'node:test' + +import { isLvmPartitionType, isProbeableForLvm, lvmOnlyDevice, toLvPartitions } from './_fileRestore.mjs' +import { + LINUX_DATA_PARTITION_TYPE_GPT, + LINUX_DATA_PARTITION_TYPE_MBR, + LVM_PARTITION_TYPE_GPT, + LVM_PARTITION_TYPE_MBR, +} from './_listPartitions.mjs' + +// well-known GPT type GUIDs that must NOT be probed/treated as LVM +const EFI = 'c12a7328-f81f-11d2-ba4b-00a0c93ec93b' +const BIOS_BOOT = '21686148-6449-6e6f-744e-656564454649' +const SWAP = '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f' + +test('lvmOnlyDevice builds an accept-only global_filter for the device', () => { + assert.equal( + lvmOnlyDevice('/dev/mapper/xo-pv-abcd1234'), + 'devices { global_filter=[ "a|^/dev/mapper/xo-pv-abcd1234$|", "r|.*|" ] }' + ) + assert.equal(lvmOnlyDevice('/dev/loop7'), 'devices { global_filter=[ "a|^/dev/loop7$|", "r|.*|" ] }') +}) + +test('isLvmPartitionType matches only the LVM partition types', () => { + assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_MBR), true) + assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_GPT), true) + assert.equal(isLvmPartitionType(LINUX_DATA_PARTITION_TYPE_GPT), false) + assert.equal(isLvmPartitionType(EFI), false) + assert.equal(isLvmPartitionType(undefined), false) +}) + +test('isProbeableForLvm matches only generic Linux-data types', () => { + assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_MBR), true) + assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_GPT), true) + // never probe these + assert.equal(isProbeableForLvm(LVM_PARTITION_TYPE_GPT), false) + assert.equal(isProbeableForLvm(BIOS_BOOT), false) + assert.equal(isProbeableForLvm(EFI), false) + assert.equal(isProbeableForLvm(SWAP), false) + assert.equal(isProbeableForLvm(undefined), false) +}) + +test('toLvPartitions builds entries, skips unnamed LVs, shows the original VG name', () => { + const lvItems = [ + { lv_name: 'ubuntu-lv', lv_path: '/dev/xohash/ubuntu-lv', lv_size: '1000', vg_name: 'xohash' }, + // a PV with no LV yields a row with an empty lv_name → must be skipped + { lv_name: '', lv_path: '', lv_size: '0', vg_name: 'xohash' }, + { lv_name: 'swap_1', lv_path: '/dev/xohash/swap_1', lv_size: '500', vg_name: 'xohash' }, + ] + assert.deepEqual(toLvPartitions('part3', 'ubuntu-vg', lvItems), [ + { id: 'part3/xohash/ubuntu-lv', name: 'ubuntu-vg/ubuntu-lv', size: '1000' }, + { id: 'part3/xohash/swap_1', name: 'ubuntu-vg/swap_1', size: '500' }, + ]) +}) + +test('toLvPartitions falls back to the bare LV name when the original VG name is unknown', () => { + const lvItems = [{ lv_name: 'data', lv_path: '/dev/xohash/data', lv_size: '42', vg_name: 'xohash' }] + // raw-disk PV → empty partitionId + assert.deepEqual(toLvPartitions('', undefined, lvItems), [{ id: '/xohash/data', name: 'data', size: '42' }]) +}) diff --git a/@xen-orchestra/backups/_listPartitions.mjs b/@xen-orchestra/backups/_listPartitions.mjs index 2897bdbe8e..c9a5f4dde4 100644 --- a/@xen-orchestra/backups/_listPartitions.mjs +++ b/@xen-orchestra/backups/_listPartitions.mjs @@ -29,6 +29,15 @@ export const LVM_PARTITION_TYPE_MBR = 0x8e // GPT LVM type export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928' +// Generic "Linux filesystem data" types. These are the only non-LVM-typed +// partitions worth probing for a hidden LVM PV, because some installers (e.g. +// Ubuntu subiquity) place a PV on a partition left with this generic type. +// Other types (BIOS boot, EFI, swap, …) are never LVM PVs. +// MBR Linux native +export const LINUX_DATA_PARTITION_TYPE_MBR = 0x83 +// GPT Linux filesystem data +export const LINUX_DATA_PARTITION_TYPE_GPT = '0fc63daf-8483-4772-8e79-3d69d8477de4' + const parsePartxLine = createParser({ keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()), valueTransform: (value, key) => { diff --git a/@xen-orchestra/qa-test/package.json b/@xen-orchestra/qa-test/package.json index cee4d52463..980fa30d96 100644 --- a/@xen-orchestra/qa-test/package.json +++ b/@xen-orchestra/qa-test/package.json @@ -43,6 +43,7 @@ "qa:backup:metadata": "node --env-file-if-exists=.env --test tests/metadata-backup.test.js", "demo": "node --env-file-if-exists=.env index.js", "report": "node --env-file-if-exists=.env scripts/run-and-report.mjs", - "report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test" + "report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test", + "qa:backup:file-restore": "node --env-file-if-exists=.env --test tests/backup.file-restore.test.js" } } diff --git a/@xen-orchestra/qa-test/tests/backup.file-restore.test.js b/@xen-orchestra/qa-test/tests/backup.file-restore.test.js new file mode 100644 index 0000000000..22b060f8ea --- /dev/null +++ b/@xen-orchestra/qa-test/tests/backup.file-restore.test.js @@ -0,0 +1,181 @@ +import assert from 'node:assert' +import { after, before, describe, it } from 'node:test' +import { createLogger } from '@xen-orchestra/log' + +import { backupConfig } from '../backup.config.js' +import { generateBackupJobName, getDefaultSchedule, getScheduleKey } from '../utils/index.js' +import { assertBackupSuccess } from '../utils/backupUtils.js' +import { setup, teardown } from './setup.js' + +const log = createLogger('qa:backup:file-restore') + +// Magic bytes used to validate archive format integrity +const TGZ_MAGIC = [0x1f, 0x8b] +const ZIP_MAGIC = [0x50, 0x4b] + +describe('Incremental backup file restore', () => { + let dispatchClient + let tracker + let vm + let backupRepository + + // The second (delta) backup to run file restore on + let incrementalBackup + + before(async () => { + ;({ dispatchClient, tracker, vm, backupRepository } = await setup()) + + const name = generateBackupJobName() + const schedule = getDefaultSchedule() + const config = backupConfig(name, schedule, vm, backupRepository) + config.mode = 'delta' + + const backupJobId = await dispatchClient.backup.createBackupJob(config) + const backupJob = await dispatchClient.backup.details(backupJobId) + + tracker.trackResource('backupJob', backupJobId, { name, mode: 'delta' }) + const scheduleKey = getScheduleKey(backupJob) + if (scheduleKey) { + tracker.trackResource('schedule', scheduleKey, { name, backupJobId }) + } + + assert(scheduleKey, 'Schedule key is required for file restore tests') + + // First run → full backup (base), second run → delta (incremental) + for (let i = 0; i < 2; i++) { + const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, scheduleKey) + assertBackupSuccess(result, `Backup run ${i + 1}/2`) + log.debug(`Completed backup run ${i + 1}/2`, { status: result.status }) + } + + // Pick the most recent delta backup for file restore testing + const backupsByRemoteAndVm = await dispatchClient.xoClient.call('backupNg.listVmBackups', { + remotes: [backupRepository.id], + }) + + const vmBackups = backupsByRemoteAndVm[backupRepository.id]?.[vm.uuid] + assert(Array.isArray(vmBackups) && vmBackups.length >= 2, 'Expected at least 2 backups after two runs') + + const sorted = [...vmBackups].sort((a, b) => a.timestamp - b.timestamp) + incrementalBackup = sorted[sorted.length - 1] + + assert.strictEqual(incrementalBackup.mode, 'delta', 'Latest backup should be a delta (incremental) backup') + assert(incrementalBackup.disks.length > 0, 'Incremental backup should contain at least one disk') + + log.debug('Selected incremental backup for file restore', { + backupId: incrementalBackup.id, + diskCount: incrementalBackup.disks.length, + timestamp: new Date(incrementalBackup.timestamp).toISOString(), + }) + }) + + after(async () => { + await teardown(dispatchClient, tracker) + }) + + it('should list partitions, list files, and restore first 2 files in tgz and zip for each disk', async () => { + const remoteId = backupRepository.id + + for (const disk of incrementalBackup.disks) { + log.debug('Processing disk', { diskId: disk.id, diskName: disk.name }) + + const partitions = await dispatchClient.xoClient.call('backupNg.listPartitions', { + remote: remoteId, + disk: disk.id, + }) + + assert(Array.isArray(partitions), `listPartitions should return an array for disk ${disk.id}`) + log.debug('Found partitions', { diskId: disk.id, count: partitions.length }) + + // When listPartitions returns empty, treat the disk as a single raw (unpartitioned) volume. + // The API accepts partition: undefined for this case. + const partitionTargets = partitions.length > 0 ? partitions : [{ id: undefined }] + + for (const partition of partitionTargets) { + const partitionId = partition.id + log.debug('Processing partition', { diskId: disk.id, partitionId }) + + let fileEntries + try { + fileEntries = await dispatchClient.xoClient.call('backupNg.listFiles', { + remote: remoteId, + disk: disk.id, + ...(partitionId !== undefined && { partition: partitionId }), + path: '/', + }) + } catch (error) { + // Some partitions (swap, EFI system) cannot be mounted — skip gracefully + log.warn('Could not list files for partition, skipping', { + diskId: disk.id, + partitionId, + error: error.message, + }) + continue + } + + assert( + fileEntries !== null && typeof fileEntries === 'object', + `listFiles should return an object for disk ${disk.id} partition ${partitionId}` + ) + + // Directories are suffixed with '/', files are not + const fileNames = Object.keys(fileEntries).filter(name => !name.endsWith('/')) + + if (fileNames.length === 0) { + log.debug('No files at partition root, skipping', { diskId: disk.id, partitionId }) + continue + } + + // Restore at most 2 files — paths must be absolute for fetchFiles + const paths = fileNames.slice(0, 2).map(name => `/${name}`) + + log.debug('Restoring files', { diskId: disk.id, partitionId, paths }) + + for (const format of ['tgz', 'zip']) { + await assertFetchFiles(dispatchClient, { remote: remoteId, disk: disk.id, partitionId, paths, format }) + } + } + } + }) +}) + +/** + * Calls backupNg.fetchFiles, downloads the resulting archive, and validates + * that the response is a non-empty, correctly-formatted archive. + * + * @param {import('../client/dispatchClient.js').DispatchClient} dispatchClient + * @param {{ remote: string, disk: string, partitionId: string|undefined, paths: string[], format: 'tgz'|'zip' }} opts + */ +async function assertFetchFiles(dispatchClient, { remote, disk, partitionId, paths, format }) { + const params = { + remote, + disk, + paths, + format, + ...(partitionId !== undefined && { partition: partitionId }), + } + + const result = await dispatchClient.xoClient.call('backupNg.fetchFiles', params) + const downloadPath = result.$getFrom + assert( + typeof downloadPath === 'string' && downloadPath.length > 0, + `fetchFiles (${format}) should return a non-empty $getFrom URL` + ) + + const response = await fetch(`${dispatchClient.restApiClient.baseUrl}${downloadPath}`, { + method: 'GET', + headers: dispatchClient.restApiClient.headers, + signal: AbortSignal.timeout(120_000), + }) + + assert(response.ok, `Download failed for ${format}: HTTP ${response.status} ${response.statusText}`) + + const bytes = Buffer.from(await response.arrayBuffer()) + assert(bytes.length > 0, `Downloaded ${format} archive should not be empty`) + + const magic = format === 'tgz' ? TGZ_MAGIC : ZIP_MAGIC + assert.strictEqual(bytes[0], magic[0], `${format} archive should start with correct magic byte[0]`) + assert.strictEqual(bytes[1], magic[1], `${format} archive should start with correct magic byte[1]`) + + log.debug('File restore validated', { format, size: bytes.length, paths, partitionId }) +} diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 17dc3267f8..9441fc0272 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -64,6 +64,8 @@ - [Rest Api] Fix `possibly unhandled rejection invalid crendentials` (PR [#9938](https://github.com/vatesfr/xen-orchestra/pull/9938)) - [Backups] Fixed "Cannot read properties of undefined" issues (PR [#9944](https://github.com/vatesfr/xen-orchestra/pull/9944)) - [REST API] `GET /vms/:id.:format`, `GET /vm-templates/:id.:format`, `GET /vm-snapshots/:id.:format` now correctly support explicit compress query param (`zstd` | `gzip`). Still support `true` | `false` as deprecated value (PR [#9960](https://github.com/vatesfr/xen-orchestra/pull/9960)) +- [Backup/Restore] Fix file-level restore of VMs whose disks use LVM (e.g. the default Ubuntu install layout): logical volumes are now listed and can be restored, including when restoring several copies of the same VM at once — previously failed with `unknown filesystem type 'LVM2_member'` (PR [#9776](https://github.com/vatesfr/xen-orchestra/pull/9776)) +- [Backup/Restore] Fix file-level restore hanging when downloading large folders, and high memory use when downloading a folder as a zip (PR [#9776](https://github.com/vatesfr/xen-orchestra/pull/9776)) ### Packages to release @@ -81,11 +83,13 @@ +- @vates/fuse-vhd patch - @vates/types minor - @xen-orchestra/acl minor - @xen-orchestra/backup-archive patch +- @xen-orchestra/backups patch - @xen-orchestra/disk-transform patch -- @xen-orchestra/qa-test patch +- @xen-orchestra/qa-test minor - @xen-orchestra/qcow2 patch - @xen-orchestra/rest-api minor - @xen-orchestra/web minor diff --git a/packages/xo-web/src/xo-app/backup/file-restore/restore-file-modal.js b/packages/xo-web/src/xo-app/backup/file-restore/restore-file-modal.js index 1f43a346cb..fd78c23ca9 100644 --- a/packages/xo-web/src/xo-app/backup/file-restore/restore-file-modal.js +++ b/packages/xo-web/src/xo-app/backup/file-restore/restore-file-modal.js @@ -18,9 +18,23 @@ import { listPartitions, listFiles } from 'xo' // ----------------------------------------------------------------------------- const PARTITION_TYPE_NAMES = { + // MBR types + 0x05: 'Extended', 0x07: 'NTFS', - 0x0c: 'FAT', - 0x83: 'LINUX', + 0x0b: 'FAT32', + 0x0c: 'FAT32', + 0x0e: 'FAT16', + 0x82: 'Linux swap', + 0x83: 'Linux', + 0x8e: 'LVM', + 0xfd: 'Linux RAID', + // GPT types + '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': 'Linux swap', + '0fc63daf-8483-4772-8e79-3d69d8477de4': 'Linux', + '21686148-6449-6e6f-744e-656564454649': 'BIOS boot', + 'c12a7328-f81f-11d2-ba4b-00a0c93ec93b': 'EFI System', + 'e6d6d379-f507-44c2-a23c-238f2a3df928': 'LVM', + 'ebd0a0a2-b9e5-4433-87c0-68b6b72699c7': 'Windows', } const BACKUP_RENDERER = getRenderXoItemOfType('backup') @@ -31,12 +45,16 @@ const diskOptionRenderer = disk => ( ) -const partitionOptionRenderer = partition => ( - - {partition.name} {defined(PARTITION_TYPE_NAMES[partition.type], partition.type)}{' '} - {partition.size && `(${formatSize(+partition.size)})`} - -) +const partitionOptionRenderer = partition => { + const typeName = defined(PARTITION_TYPE_NAMES[partition.type], partition.type) + const label = [partition.name, typeName].filter(Boolean).join(' ') + return ( + + {label} + {partition.size && ` (${formatSize(+partition.size)})`} + + ) +} const fileOptionRenderer = ({ isFile, name }) => (