mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-09-10 22:14:48 -05:00
feat(backup): implement synchronized snapshot option back (#10136)
Adds a synchronizedSnapshot backup setting. When enabled, all of a job's VM snapshots are taken up front in a batch before any transfer begins, and each VM's backup then reuses its pre-taken snapshot instead of snapshotting again.
Settings:
synchronizedSnapshot:
false (default)
true (synchronize every VM in the job)
<tag> (synchronize only VMs carrying that tag, others are backed up normally).
snapshotConcurrency: max concurrent snapshots during the batch phase (default 2).
A batch member that fails to open is silently dropped from synchronization and backed up normally.
Unit tested where applicable and e2e tests added in qa-test.
Checkbox added in xo5 backup jobs to synchronize snapshots, tags not supported yet.
This commit is contained in:
@@ -9,7 +9,9 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
|
||||
import { FullXapi } from './_vmRunners/FullXapi.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -31,7 +33,9 @@ const DEFAULT_XAPI_VM_SETTINGS = {
|
||||
nRetriesVmBackupFailures: 0,
|
||||
offlineBackup: false,
|
||||
offlineSnapshot: false,
|
||||
snapshotConcurrency: 2,
|
||||
snapshotRetention: 0,
|
||||
synchronizedSnapshot: false,
|
||||
timeout: 0,
|
||||
useNbd: false,
|
||||
unconditionalSnapshot: false,
|
||||
@@ -47,6 +51,16 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
return baseSettings
|
||||
}
|
||||
|
||||
_getVmBackup(jobMode, opts) {
|
||||
if (jobMode === 'delta') {
|
||||
return new IncrementalXapi(opts)
|
||||
} else if (jobMode === 'full') {
|
||||
return new FullXapi(opts)
|
||||
}
|
||||
|
||||
throw new Error(`Job mode ${jobMode} not implemented`)
|
||||
}
|
||||
|
||||
async run() {
|
||||
const job = this._job
|
||||
|
||||
@@ -95,7 +109,81 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
const allSettings = this._job.settings
|
||||
const baseSettings = this._baseSettings
|
||||
|
||||
const queue = new Set(vmIds)
|
||||
const preTakenTimestampByVmId = {}
|
||||
const failedSnapshotByVmId = {}
|
||||
if (settings.synchronizedSnapshot) {
|
||||
await Task.run({ properties: { name: 'snapshot VMs' } }, async () => {
|
||||
await Disposable.use(
|
||||
Disposable.all(vmIds.map(vmId => this._getRecord('VM', vmId).catch(noop))),
|
||||
async vms => {
|
||||
// remove vms that failed (already handled)
|
||||
vms = vms.filter(_ => _ !== undefined)
|
||||
|
||||
const batchIds = selectSynchronizedSnapshotVms(settings.synchronizedSnapshot, vms)
|
||||
|
||||
await asyncEach(
|
||||
[...vms].filter(vm => batchIds.has(vm.uuid)),
|
||||
async vm => {
|
||||
const vmSettings = { ...settings, ...allSettings[vm.uuid] }
|
||||
const opts = {
|
||||
baseSettings,
|
||||
config,
|
||||
getSnapshotNameLabel,
|
||||
healthCheckSr,
|
||||
job,
|
||||
remoteAdapters,
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
srs,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}
|
||||
|
||||
let vmBackup
|
||||
try {
|
||||
vmBackup = this._getVmBackup(job.mode, opts)
|
||||
if (
|
||||
!vmBackup._settings.offlineBackup &&
|
||||
!vmBackup._settings.offlineSnapshot &&
|
||||
(await vmBackup._mustDoSnapshot())
|
||||
) {
|
||||
await vmBackup._prepareAndSnapshot()
|
||||
preTakenTimestampByVmId[vm.uuid] = vmBackup.timestamp
|
||||
}
|
||||
} catch (error) {
|
||||
failedSnapshotByVmId[vm.uuid] = error
|
||||
if (vmBackup !== undefined) {
|
||||
try {
|
||||
await vmBackup._fetchJobSnapshots()
|
||||
await vmBackup._removeUnusedSnapshots()
|
||||
await vmBackup._cleanMetadata()
|
||||
} catch (cleanupError) {
|
||||
// Best effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
concurrency: settings.snapshotConcurrency,
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
const snapshotedVmIds = new Set(vmIds.filter(id => !(id in failedSnapshotByVmId)))
|
||||
Object.entries(failedSnapshotByVmId).forEach(([vmId, error]) => {
|
||||
Task.run(
|
||||
{
|
||||
properties: { id: vmId, name: 'backup VM', type: 'VM' },
|
||||
},
|
||||
() => Promise.reject(error)
|
||||
).catch(noop)
|
||||
})
|
||||
|
||||
const queue = new Set(snapshotedVmIds)
|
||||
const taskByVmId = {}
|
||||
const nTriesByVmId = {}
|
||||
|
||||
@@ -155,21 +243,16 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
srs,
|
||||
// when set, the batch phase already snapshotted this VM; the
|
||||
// runner re-finds that snapshot by its metadata (see _snapshot)
|
||||
synchronizedSnapshotTimestamp: preTakenTimestampByVmId[vmUuid],
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}
|
||||
|
||||
let vmBackup
|
||||
if (job.mode === 'delta') {
|
||||
vmBackup = new IncrementalXapi(opts)
|
||||
} else {
|
||||
if (job.mode === 'full') {
|
||||
vmBackup = new FullXapi(opts)
|
||||
} else {
|
||||
throw new Error(`Job mode ${job.mode} not implemented`)
|
||||
}
|
||||
}
|
||||
const vmBackup = this._getVmBackup(job.mode, opts)
|
||||
|
||||
return vmBackup.run().catch(error => {
|
||||
taskError = error
|
||||
})
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
export function selectSynchronizedSnapshotVms(synchronizedSnapshot, vms) {
|
||||
if (synchronizedSnapshot === false) {
|
||||
return new Set()
|
||||
}
|
||||
|
||||
let matches
|
||||
if (typeof synchronizedSnapshot === 'string') {
|
||||
matches = vms.filter(vm => vm.tags.includes(synchronizedSnapshot))
|
||||
} else if (synchronizedSnapshot === true) {
|
||||
matches = vms
|
||||
} else {
|
||||
Task.warning('unsupported synchronizedSnapshot option, expected a boolean or a tag name', { synchronizedSnapshot })
|
||||
return new Set()
|
||||
}
|
||||
|
||||
return new Set(matches.map(vm => vm.uuid))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
|
||||
|
||||
// Helper mirroring the minimal VM record shape the selector needs.
|
||||
const vm = (uuid, tags = []) => ({ uuid, tags })
|
||||
|
||||
// `selectSynchronizedSnapshotVms(synchronizedSnapshot, vms)` must return a Set
|
||||
// of the UUIDs that should be snapshotted together in the batch phase.
|
||||
// - false -> feature off, no VM is batched
|
||||
// - true -> every VM is batched, regardless of tags
|
||||
// - '<tagName>' -> only VMs carrying that exact tag are batched
|
||||
const tests = [
|
||||
{
|
||||
label: 'false disables batching',
|
||||
setting: false,
|
||||
vms: [vm('a', ['prod']), vm('b')],
|
||||
expected: [],
|
||||
},
|
||||
{
|
||||
label: 'true batches every VM regardless of tags',
|
||||
setting: true,
|
||||
vms: [vm('a', ['prod']), vm('b')],
|
||||
expected: ['a', 'b'],
|
||||
},
|
||||
{
|
||||
label: 'tag batches only the VMs carrying it',
|
||||
setting: 'prod',
|
||||
vms: [vm('a', ['prod']), vm('b', ['dev']), vm('c', ['prod', 'other'])],
|
||||
expected: ['a', 'c'],
|
||||
},
|
||||
{
|
||||
label: 'tag carried by no VM yields an empty batch',
|
||||
setting: 'prod',
|
||||
vms: [vm('a', ['dev']), vm('b')],
|
||||
expected: [],
|
||||
},
|
||||
{
|
||||
label: 'tag carried by every VM batches all of them',
|
||||
setting: 'prod',
|
||||
vms: [vm('a', ['prod']), vm('b', ['prod'])],
|
||||
expected: ['a', 'b'],
|
||||
},
|
||||
{
|
||||
label: 'empty VM list yields an empty batch',
|
||||
setting: true,
|
||||
vms: [],
|
||||
expected: [],
|
||||
},
|
||||
{
|
||||
label: 'empty-string tag matches no VM when none carries an empty tag',
|
||||
setting: '',
|
||||
vms: [vm('a', ['prod'])],
|
||||
expected: [],
|
||||
},
|
||||
{
|
||||
label: 'non boolean or string yiels an empty batch',
|
||||
setting: 1,
|
||||
vms: [vm('a', ['prod'])],
|
||||
expected: [],
|
||||
},
|
||||
]
|
||||
|
||||
describe('selectSynchronizedSnapshotVms()', () => {
|
||||
for (const { label, setting, vms, expected } of tests) {
|
||||
it(label, () => {
|
||||
const result = selectSynchronizedSnapshotVms(setting, vms)
|
||||
assert.ok(result instanceof Set, 'result must be a Set')
|
||||
assert.deepEqual([...result].sort(), [...expected].sort())
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -80,14 +80,14 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
|
||||
const sizeContainer = watchStreamSize(stream)
|
||||
|
||||
const timestamp = Date.now()
|
||||
const transferStart = Date.now()
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.run({
|
||||
maxStreamLength,
|
||||
sizeContainer,
|
||||
stream: forkStreamUnpipe(stream),
|
||||
timestamp,
|
||||
timestamp: this.timestamp,
|
||||
vm,
|
||||
vmSnapshot: exportedVm,
|
||||
}),
|
||||
@@ -95,8 +95,7 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
)
|
||||
|
||||
const { size } = sizeContainer
|
||||
const end = Date.now()
|
||||
const duration = end - timestamp
|
||||
const duration = Date.now() - transferStart
|
||||
debug('transfer complete', {
|
||||
duration,
|
||||
speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
DELTA_CHAIN_LENGTH,
|
||||
EXPORTED_SUCCESSFULLY,
|
||||
setVmDeltaChainLength,
|
||||
markExportSuccessfull,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
@@ -68,7 +67,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
|
||||
// @todo : reimplement throttle
|
||||
|
||||
const timestamp = Date.now()
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.transfer({
|
||||
@@ -78,7 +76,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
// clean; the tip's flag lets the next run skip re-probing
|
||||
includeNonNbdQcow2Fix: true,
|
||||
isVhdDifferencing,
|
||||
timestamp,
|
||||
timestamp: this.timestamp,
|
||||
vm,
|
||||
vmSnapshot: exportedVm,
|
||||
}),
|
||||
@@ -91,11 +89,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
|
||||
}
|
||||
|
||||
// not the case if offlineBackup
|
||||
if (exportedVm.is_a_snapshot) {
|
||||
await markExportSuccessfull(this._xapi, exportedVm.$ref)
|
||||
}
|
||||
|
||||
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { asyncEach } from '@vates/async-each'
|
||||
import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { defer } from 'golike-defer'
|
||||
import { Task } from '@vates/task'
|
||||
import { formatDateTime } from '@xen-orchestra/xapi'
|
||||
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Abstract } from './_Abstract.mjs'
|
||||
@@ -16,9 +17,11 @@ import {
|
||||
JOB_ID,
|
||||
SCHEDULE_ID,
|
||||
VM_UUID,
|
||||
EXPORTED_SUCCESSFULLY,
|
||||
resetVmOtherConfig,
|
||||
setVmOtherConfig,
|
||||
setVmSnapshotContentKeys,
|
||||
markExportSuccessfull,
|
||||
} from '../../_otherConfig.mjs'
|
||||
|
||||
const { warn, info } = createLogger('xo:backups:AbstractXapi')
|
||||
@@ -36,6 +39,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings,
|
||||
srs,
|
||||
synchronizedSnapshotTimestamp,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
@@ -63,9 +67,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
|
||||
// VM (snapshot) that is really exported
|
||||
this._exportedVm = undefined
|
||||
this._synchronizedSnapshotTimestamp = synchronizedSnapshotTimestamp
|
||||
this._vm = vm
|
||||
|
||||
this._baseVdis = undefined
|
||||
this._getSnapshotNameLabel = getSnapshotNameLabel
|
||||
this._isIncremental = job.mode === 'delta'
|
||||
this._healthCheckSr = healthCheckSr
|
||||
@@ -180,6 +184,27 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
|
||||
async _snapshot() {
|
||||
if (this._synchronizedSnapshotTimestamp !== undefined) {
|
||||
const datetime = formatDateTime(this._synchronizedSnapshotTimestamp)
|
||||
const candidates = this._vm.$snapshots.filter(
|
||||
snapshot =>
|
||||
snapshot?.other_config[JOB_ID] === this._jobId &&
|
||||
snapshot?.other_config[DATETIME] === datetime &&
|
||||
!snapshot.other_config[EXPORTED_SUCCESSFULLY]
|
||||
)
|
||||
if (candidates.length === 1) {
|
||||
this._exportedVm = candidates[0]
|
||||
this.timestamp = this._synchronizedSnapshotTimestamp
|
||||
return
|
||||
}
|
||||
|
||||
warn('expected exactly one synchronized snapshot to reuse, taking a fresh one', {
|
||||
vm: this._vm.uuid,
|
||||
datetime,
|
||||
count: candidates.length,
|
||||
})
|
||||
}
|
||||
|
||||
const vm = this._vm
|
||||
const xapi = this._xapi
|
||||
|
||||
@@ -396,7 +421,20 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
await xapi.barrier()
|
||||
// ensure cached object are up to date
|
||||
this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
|
||||
const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
|
||||
let disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
|
||||
|
||||
// The synchronized snapshot for this run is taken up-front by the batch
|
||||
// phase but transferred later. Until it has been exported, hide it from
|
||||
// retention so it is neither reclaimed nor allowed to steal the delta
|
||||
// base's "most recent" protection. This makes the pre-transfer pass behave
|
||||
// like a normal run, where the snapshot does not exist yet.
|
||||
if (this._synchronizedSnapshotTimestamp !== undefined) {
|
||||
const datetime = formatDateTime(this._synchronizedSnapshotTimestamp)
|
||||
const isInFlightSyncSnapshot = ({ other_config }) =>
|
||||
other_config[DATETIME] === datetime && !other_config[EXPORTED_SUCCESSFULLY]
|
||||
this._jobSnapshotVdis = this._jobSnapshotVdis.filter(vdi => !isInFlightSyncSnapshot(vdi))
|
||||
disklessVmSnapshots = disklessVmSnapshots.filter(vm => !isInFlightSyncSnapshot(vm))
|
||||
}
|
||||
|
||||
// get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
|
||||
const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
|
||||
@@ -464,6 +502,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
vm = vdiVm
|
||||
}
|
||||
}
|
||||
|
||||
if (vm?.$ref !== undefined) {
|
||||
return xapi.VM_destroy(vm.$ref)
|
||||
} else {
|
||||
@@ -493,6 +532,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
|
||||
return
|
||||
}
|
||||
|
||||
await xapi.VM_destroy(snapshotPerDatetime[datetime])
|
||||
})
|
||||
})
|
||||
@@ -551,6 +591,11 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
|
||||
async _prepareAndSnapshot() {
|
||||
await this._cleanMetadata()
|
||||
await this._snapshot()
|
||||
}
|
||||
|
||||
async run($defer) {
|
||||
const settings = this._settings
|
||||
assert(
|
||||
@@ -614,6 +659,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
if (this._writers.size !== 0) {
|
||||
await this._copy()
|
||||
}
|
||||
// not the case if offlineBackup
|
||||
if (this._exportedVm.is_a_snapshot) {
|
||||
await markExportSuccessfull(this._xapi, this._exportedVm.$ref)
|
||||
}
|
||||
} finally {
|
||||
if (startAfter) {
|
||||
ignoreErrors.call(vm.$callAsync('start', false, false))
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { formatDateTime } from '@xen-orchestra/xapi'
|
||||
|
||||
import { AbstractXapi } from './_AbstractXapi.mjs'
|
||||
import { DATETIME, EXPORTED_SUCCESSFULLY, JOB_ID, SCHEDULE_ID } from '../../_otherConfig.mjs'
|
||||
|
||||
// Build an AbstractXapi instance without running its heavy constructor, then
|
||||
// assign only the fields the method under test reads.
|
||||
const makeRunner = props => Object.assign(Object.create(AbstractXapi.prototype), props)
|
||||
|
||||
describe('_snapshot() synchronized-snapshot reuse guard', () => {
|
||||
it('reuses the batch snapshot, found by metadata, without snapshotting again', async () => {
|
||||
let mustDoSnapshotCalls = 0
|
||||
const timestamp = 1717200000000
|
||||
// the snapshot taken by the batch phase, identified by its other_config
|
||||
const batchSnapshot = {
|
||||
$ref: 'batch-snapshot-ref',
|
||||
other_config: {
|
||||
[JOB_ID]: 'job-1',
|
||||
[SCHEDULE_ID]: 'schedule-1',
|
||||
[DATETIME]: formatDateTime(timestamp),
|
||||
},
|
||||
}
|
||||
const runner = makeRunner({
|
||||
_synchronizedSnapshotTimestamp: timestamp,
|
||||
_jobId: 'job-1',
|
||||
scheduleId: 'schedule-1',
|
||||
_vm: { uuid: 'vm-uuid', $snapshots: [batchSnapshot] },
|
||||
_xapi: { barrier: async () => {} },
|
||||
// if the metadata lookup fails, _snapshot() falls through to here
|
||||
_mustDoSnapshot: async () => {
|
||||
mustDoSnapshotCalls++
|
||||
return false
|
||||
},
|
||||
})
|
||||
|
||||
await runner._snapshot()
|
||||
|
||||
assert.equal(mustDoSnapshotCalls, 0, '_mustDoSnapshot() should not be called when the batch snapshot is found')
|
||||
assert.equal(runner._exportedVm, batchSnapshot, '_exportedVm should be the batch snapshot found by metadata')
|
||||
assert.equal(runner.timestamp, timestamp, 'timestamp should be the synchronized snapshot timestamp')
|
||||
})
|
||||
|
||||
it('takes a fresh snapshot when the batch snapshot can no longer be found', async () => {
|
||||
let mustDoSnapshotCalls = 0
|
||||
const vm = { uuid: 'vm-uuid', $snapshots: [] } // the pre-taken snapshot is gone
|
||||
const runner = makeRunner({
|
||||
_synchronizedSnapshotTimestamp: 1717200000000,
|
||||
_jobId: 'job-1',
|
||||
scheduleId: 'schedule-1',
|
||||
_vm: vm,
|
||||
_xapi: { barrier: async () => {} },
|
||||
_settings: {},
|
||||
_mustDoSnapshot: async () => {
|
||||
mustDoSnapshotCalls++
|
||||
return false
|
||||
},
|
||||
})
|
||||
|
||||
await runner._snapshot()
|
||||
|
||||
assert.equal(
|
||||
mustDoSnapshotCalls,
|
||||
1,
|
||||
'a missing batch snapshot should not be reused: _snapshot() should fall through and take a fresh snapshot'
|
||||
)
|
||||
assert.equal(
|
||||
runner._exportedVm,
|
||||
vm,
|
||||
'_exportedVm should fall back to the freshly-snapshotted VM, not stay on a stale/missing batch snapshot'
|
||||
)
|
||||
})
|
||||
|
||||
it('without a pre-taken snapshot and no snapshot needed, exports the live VM', async () => {
|
||||
let mustDoSnapshotCalls = 0
|
||||
const vm = { uuid: 'vm-uuid' }
|
||||
const runner = makeRunner({
|
||||
_exportedVm: undefined,
|
||||
_vm: vm,
|
||||
_xapi: {},
|
||||
_settings: {},
|
||||
_mustDoSnapshot: async () => {
|
||||
mustDoSnapshotCalls++
|
||||
return false
|
||||
},
|
||||
})
|
||||
|
||||
await runner._snapshot()
|
||||
|
||||
assert.equal(mustDoSnapshotCalls, 1, '_mustDoSnapshot() should be called once when no snapshot was pre-taken')
|
||||
assert.equal(runner._exportedVm, vm, '_exportedVm should be set to the live VM when no snapshot is needed')
|
||||
assert.equal(typeof runner.timestamp, 'number', 'timestamp should be set when exporting the live VM')
|
||||
})
|
||||
})
|
||||
|
||||
describe('_removeUnusedSnapshots() protects the pre-taken synchronized snapshot', () => {
|
||||
const OLD_DATETIME = '20240101T00:00:00Z'
|
||||
const SYNC_TIMESTAMP = 1717200000000
|
||||
const SYNC_DATETIME = formatDateTime(SYNC_TIMESTAMP)
|
||||
|
||||
// Full-mode job with snapshotRetention 0, so retention wants to remove every
|
||||
// snapshot. `vdi-fresh` is the snapshot taken up-front by the synchronized
|
||||
// batch phase (identified by SYNC_TIMESTAMP); until it is exported it must be
|
||||
// hidden from retention.
|
||||
const makeRemoveRunner = ({ exported }) => {
|
||||
const destroyed = []
|
||||
|
||||
const snapshotVm = ($ref, name_label) => ({
|
||||
$ref,
|
||||
name_label,
|
||||
is_control_domain: false,
|
||||
$snapshot_of: 'live-vm-ref',
|
||||
other_config: {},
|
||||
})
|
||||
const freshSnapshotVm = snapshotVm('vm-fresh', 'fresh')
|
||||
const oldSnapshotVm = snapshotVm('vm-old', 'old')
|
||||
|
||||
const vdi = ($ref, datetime, snapshotVmRecord, isExported = false) => ({
|
||||
$ref,
|
||||
other_config: {
|
||||
[DATETIME]: datetime,
|
||||
[SCHEDULE_ID]: 'schedule-1',
|
||||
...(isExported ? { [EXPORTED_SUCCESSFULLY]: 'true' } : {}),
|
||||
},
|
||||
$VBDs: [{ $VM: snapshotVmRecord }],
|
||||
})
|
||||
// the old snapshot was exported by a previous run
|
||||
const oldVdi = vdi('vdi-old', OLD_DATETIME, oldSnapshotVm, true)
|
||||
const freshVdi = vdi('vdi-fresh', SYNC_DATETIME, freshSnapshotVm, exported)
|
||||
|
||||
const registry = { 'vdi-old': oldVdi, 'vdi-fresh': freshVdi }
|
||||
|
||||
const runner = makeRunner({
|
||||
_synchronizedSnapshotTimestamp: SYNC_TIMESTAMP,
|
||||
_vm: { uuid: 'live-uuid', $snapshots: [] },
|
||||
_baseSettings: { snapshotRetention: 0 },
|
||||
_jobSnapshotVdis: [oldVdi, freshVdi],
|
||||
_disklessJobSnapshotVms: [],
|
||||
job: { mode: 'full', settings: {} },
|
||||
_xapi: {
|
||||
barrier: async () => {},
|
||||
getObject: ref => registry[ref],
|
||||
VM_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
VDI_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
return { runner, destroyed }
|
||||
}
|
||||
|
||||
it('does not destroy the pre-taken snapshot before it has been transferred', async () => {
|
||||
const { runner, destroyed } = makeRemoveRunner({ exported: false })
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(destroyed, ['vm-old'], 'only the older snapshot should be removed')
|
||||
})
|
||||
|
||||
it('destroys the snapshot once it has been transferred (retention 0, no leak)', async () => {
|
||||
const { runner, destroyed } = makeRemoveRunner({ exported: true })
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(destroyed.sort(), ['vm-fresh', 'vm-old'], 'both snapshots should be removed after transfer')
|
||||
})
|
||||
})
|
||||
|
||||
describe('_removeUnusedSnapshots() in the synchronized batch pre-snapshot state', () => {
|
||||
it('delta mode: keeps the base snapshot when the fresh one has not been taken yet', async () => {
|
||||
const destroyed = []
|
||||
|
||||
const BASE_DATETIME = '2024-06-01T00:00:00Z'
|
||||
const baseSnapshotVm = {
|
||||
$ref: 'vm-base',
|
||||
name_label: 'base',
|
||||
is_control_domain: false,
|
||||
$snapshot_of: 'live-vm-ref',
|
||||
// exported successfully by the previous run
|
||||
other_config: { [EXPORTED_SUCCESSFULLY]: 'true' },
|
||||
}
|
||||
const baseVdi = {
|
||||
$ref: 'vdi-base',
|
||||
other_config: { [DATETIME]: BASE_DATETIME, [SCHEDULE_ID]: 'schedule-1' },
|
||||
$VBDs: [{ $VM: baseSnapshotVm }],
|
||||
}
|
||||
const registry = { 'vdi-base': baseVdi }
|
||||
|
||||
const runner = makeRunner({
|
||||
// batch pre-snapshot state: the synchronized snapshot has not been taken yet
|
||||
_exportedVm: undefined,
|
||||
_vm: { uuid: 'live-uuid', $snapshots: [] },
|
||||
_baseSettings: { snapshotRetention: 0 },
|
||||
_jobSnapshotVdis: [baseVdi],
|
||||
_disklessJobSnapshotVms: [],
|
||||
job: { mode: 'delta', settings: {} },
|
||||
_xapi: {
|
||||
barrier: async () => {},
|
||||
getObject: ref => registry[ref],
|
||||
VM_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
VDI_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(destroyed, [], 'the base snapshot must be kept as the delta base for the upcoming transfer')
|
||||
})
|
||||
})
|
||||
|
||||
describe('_removeUnusedSnapshots() reclaims orphan / CBT snapshot VDIs (no attached VM)', () => {
|
||||
// Guards the `else` branch: snapshot VDIs that are not attached to any user VM
|
||||
// (e.g. CBT metadata, orphans) must be reclaimed via VDI_destroy, not VM_destroy.
|
||||
it('destroys snapshot VDIs directly when they are not attached to any VM', async () => {
|
||||
const vmDestroyed = []
|
||||
const vdiDestroyed = []
|
||||
|
||||
const orphanVdi = {
|
||||
$ref: 'vdi-orphan',
|
||||
other_config: { [DATETIME]: '2024-01-01T00:00:00Z', [SCHEDULE_ID]: 'schedule-1' },
|
||||
$VBDs: [], // not attached to any VM
|
||||
}
|
||||
const registry = { 'vdi-orphan': orphanVdi }
|
||||
|
||||
const runner = makeRunner({
|
||||
_exportedVm: undefined,
|
||||
_vm: { uuid: 'live-uuid', $snapshots: [] },
|
||||
_baseSettings: { snapshotRetention: 0 },
|
||||
_jobSnapshotVdis: [orphanVdi],
|
||||
_disklessJobSnapshotVms: [],
|
||||
job: { mode: 'full', settings: {} },
|
||||
_xapi: {
|
||||
barrier: async () => {},
|
||||
getObject: ref => registry[ref],
|
||||
VM_destroy: async ref => {
|
||||
vmDestroyed.push(ref)
|
||||
},
|
||||
VDI_destroy: async ref => {
|
||||
vdiDestroyed.push(ref)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(vdiDestroyed, ['vdi-orphan'], 'the orphan VDI should be reclaimed via VDI_destroy')
|
||||
assert.deepEqual(vmDestroyed, [], 'no VM should be destroyed for an orphan VDI')
|
||||
})
|
||||
})
|
||||
|
||||
describe('_removeUnusedSnapshots() diskless VM snapshots', () => {
|
||||
const OLD_DATETIME = '20240101T00:00:00Z'
|
||||
const SYNC_TIMESTAMP = 1717200000000
|
||||
const SYNC_DATETIME = formatDateTime(SYNC_TIMESTAMP)
|
||||
|
||||
// A diskless VM's backup snapshots are tracked as VM snapshots (no VDIs to
|
||||
// anchor them). `dl-fresh` is the snapshot the synchronized batch pre-took,
|
||||
// identified by SYNC_TIMESTAMP.
|
||||
const makeDisklessRunner = ({ exported, mode = 'full' }) => {
|
||||
const destroyed = []
|
||||
|
||||
const oldSnap = { $ref: 'dl-old', other_config: { [DATETIME]: OLD_DATETIME, [SCHEDULE_ID]: 'schedule-1' } }
|
||||
const freshSnap = { $ref: 'dl-fresh', other_config: { [DATETIME]: SYNC_DATETIME, [SCHEDULE_ID]: 'schedule-1' } }
|
||||
if (exported) freshSnap.other_config[EXPORTED_SUCCESSFULLY] = 'true'
|
||||
const registry = { 'dl-old': oldSnap, 'dl-fresh': freshSnap }
|
||||
|
||||
const runner = makeRunner({
|
||||
_synchronizedSnapshotTimestamp: SYNC_TIMESTAMP,
|
||||
_vm: { uuid: 'live-uuid', $snapshots: [] },
|
||||
_baseSettings: { snapshotRetention: 0 },
|
||||
_jobSnapshotVdis: [],
|
||||
_disklessJobSnapshotVms: [oldSnap, freshSnap],
|
||||
job: { mode, settings: {} },
|
||||
_xapi: {
|
||||
barrier: async () => {},
|
||||
getObject: ref => registry[ref],
|
||||
VM_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
VDI_destroy: async ref => {
|
||||
destroyed.push(ref)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
return { runner, destroyed }
|
||||
}
|
||||
|
||||
it('does not destroy the pre-taken diskless snapshot before it has been transferred', async () => {
|
||||
const { runner, destroyed } = makeDisklessRunner({ exported: false })
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(destroyed, ['dl-old'], 'only the older diskless snapshot should be removed')
|
||||
})
|
||||
|
||||
it('destroys the diskless snapshot once it has been transferred (retention 0, no leak)', async () => {
|
||||
const { runner, destroyed } = makeDisklessRunner({ exported: true })
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(
|
||||
destroyed.sort(),
|
||||
['dl-fresh', 'dl-old'],
|
||||
'both diskless snapshots should be removed after transfer'
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps the most recent diskless snapshot in delta mode (base for next delta)', async () => {
|
||||
// Even after transfer (exported), delta mode must retain the latest
|
||||
// snapshot so the next run can compute its delta against it.
|
||||
const { runner, destroyed } = makeDisklessRunner({ exported: true, mode: 'delta' })
|
||||
|
||||
await runner._removeUnusedSnapshots()
|
||||
|
||||
assert.deepEqual(destroyed, ['dl-old'], 'the most recent diskless snapshot should be kept in delta mode')
|
||||
})
|
||||
})
|
||||
@@ -11,7 +11,7 @@
|
||||
* @param {number} [options.nbdConcurrency] - Number of concurrent NBD connections (default: 1)
|
||||
* @returns {Object} Complete backup job configuration
|
||||
*/
|
||||
export function backupConfig(name, schedule, vm, backupRepository, options = {}) {
|
||||
export function backupConfig(name, schedule, vms, backupRepository, options = {}) {
|
||||
const config = {
|
||||
name,
|
||||
mode: 'delta', // Will be overridden by tests
|
||||
@@ -26,9 +26,7 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {})
|
||||
bypassVdiChainsCheck: true,
|
||||
},
|
||||
},
|
||||
vms: {
|
||||
[vm.uuid]: vm,
|
||||
},
|
||||
vms: Array.isArray(vms) ? Object.fromEntries(vms.map(vm => [vm.uuid, vm])) : { [vms.uuid]: vms },
|
||||
remotes: {
|
||||
[backupRepository.id]: backupRepository,
|
||||
},
|
||||
@@ -48,6 +46,14 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {})
|
||||
config.settings[''].nbdConcurrency = options.nbdConcurrency
|
||||
}
|
||||
|
||||
if (options.snapshotConcurrency !== undefined) {
|
||||
config.settings[''].snapshotConcurrency = options.snapshotConcurrency
|
||||
}
|
||||
|
||||
if (options.synchronizedSnapshot !== undefined) {
|
||||
config.settings[''].synchronizedSnapshot = options.synchronizedSnapshot
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
"qa:backup:nbd": "node --env-file-if-exists=.env --test tests/backup.nbd.test.js",
|
||||
"qa:backup:combined": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js",
|
||||
"qa:backup:combined:cycle": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='Full Cycle'",
|
||||
"qa:backup:sync": "node --env-file-if-exists=.env --test tests/backup.synchronized.test.js --test-concurrency=1",
|
||||
"qa:mirror:cr": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='CR Mode'",
|
||||
"qa:mirror": "node --env-file-if-exists=.env --test tests/backup-mirror.test.js",
|
||||
"qa:replication": "node --env-file-if-exists=.env --test --test-concurrency=1 tests/replication.test.js",
|
||||
|
||||
@@ -143,7 +143,8 @@ describe('Mirror Backup - Full Remote', () => {
|
||||
const setupResult = await setup()
|
||||
dispatchClient = setupResult.dispatchClient
|
||||
tracker = setupResult.tracker
|
||||
vm = setupResult.vm
|
||||
// Only using one VM
|
||||
vm = setupResult.vms[0]
|
||||
|
||||
assert.ok(vm, 'Setup should provide a test VM')
|
||||
log.debug('Using test VM', { name: vm.name_label, uuid: vm.uuid })
|
||||
|
||||
@@ -108,7 +108,8 @@ describe('Backup + Replication Combined Tests', () => {
|
||||
const setupResult = await setup()
|
||||
dispatchClient = setupResult.dispatchClient
|
||||
tracker = setupResult.tracker
|
||||
vm = setupResult.vm
|
||||
// Only using one VM
|
||||
vm = setupResult.vms[0]
|
||||
|
||||
assert.ok(vm, 'Setup should provide a test VM')
|
||||
log.debug('Using test VM from setup', { name: vm.name_label, uuid: vm.uuid })
|
||||
|
||||
@@ -23,7 +23,10 @@ describe('Incremental backup file restore', () => {
|
||||
let incrementalBackup
|
||||
|
||||
before(async () => {
|
||||
;({ dispatchClient, tracker, vm, backupRepository } = await setup())
|
||||
let vms
|
||||
;({ dispatchClient, tracker, vms, backupRepository } = await setup())
|
||||
// Only using one VM
|
||||
vm = vms[0]
|
||||
|
||||
const name = generateBackupJobName()
|
||||
const schedule = getDefaultSchedule()
|
||||
|
||||
@@ -22,6 +22,7 @@ const log = createLogger('qa:backup:nbd')
|
||||
|
||||
describe('NBD Incremental Backup Tests', () => {
|
||||
let vm
|
||||
let vms
|
||||
let backupRepository
|
||||
let healthCheckSr
|
||||
let backupJob
|
||||
@@ -31,7 +32,9 @@ describe('NBD Incremental Backup Tests', () => {
|
||||
let createBackupJobForTest
|
||||
|
||||
before(async () => {
|
||||
;({ dispatchClient, tracker } = await setup())
|
||||
;({ dispatchClient, tracker, vms } = await setup())
|
||||
// Only using one VM
|
||||
vm = vms[0]
|
||||
|
||||
// Look for test VMs with incremental naming pattern
|
||||
const vmPrefix = getRequiredEnv('VM_PREFIX')
|
||||
|
||||
222
@xen-orchestra/qa-test/tests/backup.synchronized.test.js
Normal file
222
@xen-orchestra/qa-test/tests/backup.synchronized.test.js
Normal file
@@ -0,0 +1,222 @@
|
||||
import assert from 'node:assert'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { backupConfig } from '../backup.config.js'
|
||||
import { FilterBuilder } from '../client/FilterBuilder.js'
|
||||
import {
|
||||
assertFullOrDelta,
|
||||
assertFullOrDeltaForSr,
|
||||
findTaskByMessage,
|
||||
generateBackupJobName,
|
||||
getDefaultSchedule,
|
||||
getScheduleKey,
|
||||
getRequiredEnv,
|
||||
} from '../utils/index.js'
|
||||
import { assertBackupSuccess, assertSynchronizedSnapshot } from '../utils/backupUtils.js'
|
||||
import { setup, teardown } from './setup.js'
|
||||
|
||||
const log = createLogger('qa:backup:base')
|
||||
|
||||
describe('Backup basic tests', () => {
|
||||
let vms
|
||||
let backupRepository
|
||||
let healthCheckSr
|
||||
let name
|
||||
let defaultSchedule
|
||||
let backupJob
|
||||
let backupJobId
|
||||
let defaultConfig
|
||||
let synchronizedConfig
|
||||
let dispatchClient
|
||||
let tracker
|
||||
let createBackupJobForTest
|
||||
|
||||
// Replica VMs created on the destination SR by the CR test; the resource
|
||||
// tracker only tracks the job, so they must be deleted explicitly.
|
||||
const replicatedVmUuids = []
|
||||
|
||||
before(async () => {
|
||||
;({ dispatchClient, tracker } = await setup({ requiredVmQty: 2 }))
|
||||
|
||||
// Look for test VMs with incremental naming pattern
|
||||
const vmPrefix = getRequiredEnv('VM_PREFIX')
|
||||
const filter = FilterBuilder.create().withGlob('name_label', `${vmPrefix}-QA-Test-*`)
|
||||
const qaVms = await dispatchClient.vm.list(filter)
|
||||
|
||||
assert(
|
||||
qaVms.length >= 2,
|
||||
`at least 2 VM with pattern "${vmPrefix}-QA-Test-*" are required - backup tests cannot run`
|
||||
)
|
||||
|
||||
// Use the first available QA VM for backup tests
|
||||
vms = qaVms.slice(0, 2)
|
||||
log.debug(
|
||||
'Found test VMs for backup sync tests',
|
||||
{ name: vms[0].name_label, uuid: vms[0].uuid },
|
||||
{ name: vms[1].name_label, uuid: vms[1].uuid }
|
||||
)
|
||||
|
||||
const backupRepositoryName = getRequiredEnv('BACKUP_REPOSITORY_NAME')
|
||||
backupRepository = await dispatchClient.backupRepository.get({ name: backupRepositoryName })
|
||||
|
||||
if (!backupRepository) {
|
||||
log.warn('Backup repository not found, creating it for tests', { name: backupRepositoryName })
|
||||
|
||||
// Create the backup repository for testing
|
||||
try {
|
||||
const backupRepositoryId = await dispatchClient.backupRepository.create(backupRepositoryName, {
|
||||
url: getRequiredEnv('BACKUP_REPOSITORY_URL'),
|
||||
})
|
||||
|
||||
// Fetch the canonical repository object from the API
|
||||
// eslint-disable-next-line require-atomic-updates -- sequential code in before() hook, no race condition
|
||||
backupRepository = await dispatchClient.backupRepository.get({ id: backupRepositoryId })
|
||||
|
||||
if (!backupRepository) {
|
||||
throw new Error(`Failed to retrieve created backup repository ${backupRepositoryId}`)
|
||||
}
|
||||
|
||||
// Track the newly created repository for cleanup
|
||||
tracker.trackResource('backupRepository', backupRepositoryId, { name: backupRepositoryName })
|
||||
} catch (error) {
|
||||
log.warn('Failed to create test backup repository', { error })
|
||||
assert.fail(
|
||||
`Backup repository "${backupRepositoryName}" is required for backup tests - could not create it: ${error.message}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Get SR for health checks by ID
|
||||
const srId = getRequiredEnv('SR_ID')
|
||||
|
||||
log.debug('Getting SR for health checks', { srId })
|
||||
healthCheckSr = await dispatchClient.sr.details(srId)
|
||||
|
||||
if (!healthCheckSr) {
|
||||
throw new Error(`SR with ID "${srId}" not found - cannot run backup tests with health checks`)
|
||||
}
|
||||
|
||||
log.debug('Found SR for health checks', { name: healthCheckSr.name_label })
|
||||
|
||||
name = generateBackupJobName()
|
||||
defaultSchedule = getDefaultSchedule()
|
||||
defaultConfig = backupConfig(name, defaultSchedule, vms, backupRepository)
|
||||
synchronizedConfig = backupConfig(name, defaultSchedule, vms, backupRepository, {
|
||||
synchronizedSnapshot: true,
|
||||
})
|
||||
|
||||
// Functions for easy tests
|
||||
createBackupJobForTest = async (config, mode) => {
|
||||
config.mode = mode
|
||||
backupJobId = await dispatchClient.backup.createBackupJob(config)
|
||||
backupJob = await dispatchClient.backup.details(backupJobId)
|
||||
assert(backupJob.mode === mode)
|
||||
|
||||
// Track the backup job and its schedule for cleanup
|
||||
tracker.trackResource('backupJob', backupJobId, { name: config.name, mode })
|
||||
|
||||
// Track the schedule (schedules are in the settings object)
|
||||
const scheduleKey = getScheduleKey(backupJob)
|
||||
if (scheduleKey) {
|
||||
tracker.trackResource('schedule', scheduleKey, { name: config.name, backupJobId })
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('Run synchronized backup jobs', () => {
|
||||
it('should run the backup job in full mode', async () => {
|
||||
await createBackupJobForTest(defaultConfig, 'full')
|
||||
const job = await dispatchClient.backup.details(backupJobId)
|
||||
const realScheduleKey = getScheduleKey(job)
|
||||
|
||||
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
|
||||
assertBackupSuccess(result, 'Synchronized full backup')
|
||||
assertFullOrDelta(result, backupRepository.id, { mustBeFull: true })
|
||||
|
||||
assert(findTaskByMessage(result, 'snapshot VMs') === null, `No batch snapshot should have been performed`)
|
||||
})
|
||||
|
||||
it('should run a synchronized delta backup and reuse the batch snapshots as the delta base on the second run', async () => {
|
||||
await createBackupJobForTest(synchronizedConfig, 'delta')
|
||||
const job = await dispatchClient.backup.details(backupJobId)
|
||||
const realScheduleKey = getScheduleKey(job)
|
||||
|
||||
// Does 2 backups, first one is full synchronized, second one is a delta.y
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
|
||||
assertBackupSuccess(result, index === 0 ? 'Synchronized full backup' : 'Synchronized delta backup')
|
||||
assertFullOrDelta(result, backupRepository.id, { mustBeFull: index === 0 })
|
||||
assertSynchronizedSnapshot(result, vms.length)
|
||||
}
|
||||
})
|
||||
|
||||
it('should replicate a synchronized delta backup and reuse the batch snapshots on the second run', async () => {
|
||||
// reuse the same SR the suite already requires; same-SR replication is fine
|
||||
const targetSrId = getRequiredEnv('SR_ID')
|
||||
|
||||
const crConfig = {
|
||||
name: generateBackupJobName(),
|
||||
mode: 'delta',
|
||||
schedules: { '': getDefaultSchedule() },
|
||||
settings: {
|
||||
'': {
|
||||
timezone: 'Europe/Paris',
|
||||
copyRetention: 3,
|
||||
preferNbd: true,
|
||||
bypassVdiChainsCheck: true,
|
||||
synchronizedSnapshot: true,
|
||||
},
|
||||
},
|
||||
vms: Object.fromEntries(vms.map(vm => [vm.uuid, vm])),
|
||||
srs: { [targetSrId]: true },
|
||||
}
|
||||
|
||||
backupJobId = await dispatchClient.backup.createBackupJob(crConfig)
|
||||
const job = await dispatchClient.backup.details(backupJobId)
|
||||
assert(job.mode === 'delta')
|
||||
tracker.trackResource('backupJob', backupJobId, { name: crConfig.name, mode: 'delta' })
|
||||
const realScheduleKey = getScheduleKey(job)
|
||||
if (realScheduleKey) {
|
||||
tracker.trackResource('schedule', realScheduleKey, { name: crConfig.name, backupJobId })
|
||||
}
|
||||
|
||||
const vmUuidsBefore = new Set((await dispatchClient.vm.list()).map(vm => vm.uuid))
|
||||
|
||||
// First run is a full replication (creates the replicas); the second must
|
||||
// come back as an incremental, which is only possible if the batch
|
||||
// snapshots survived the pre-transfer retention pass as the delta base.
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
|
||||
assertBackupSuccess(result, index === 0 ? 'Synchronized full replication' : 'Synchronized delta replication')
|
||||
assertFullOrDeltaForSr(result, targetSrId, { mustBeFull: index === 0 })
|
||||
assertSynchronizedSnapshot(result, vms.length)
|
||||
}
|
||||
|
||||
const replicas = (await dispatchClient.vm.list()).filter(vm => !vmUuidsBefore.has(vm.uuid)).map(vm => vm.uuid)
|
||||
replicatedVmUuids.push(...replicas)
|
||||
assert.strictEqual(replicas.length, vms.length, 'one replica should be created per source VM')
|
||||
})
|
||||
})
|
||||
|
||||
const cleanupVms = async vmUuids => {
|
||||
for (const vmUuid of vmUuids) {
|
||||
try {
|
||||
const vmDetails = await dispatchClient.vm.details(vmUuid)
|
||||
if (vmDetails?.power_state === 'Running') {
|
||||
await dispatchClient.vm.stop(vmUuid, { force: true })
|
||||
await dispatchClient.vm.waitForPowerState(vmUuid, 'Halted', 60_000)
|
||||
}
|
||||
await dispatchClient.vm.delete(vmUuid, { deleteDisks: true })
|
||||
log.debug('Cleaned up replica VM', { uuid: vmUuid })
|
||||
} catch (error) {
|
||||
log.warn('Failed to clean up replica VM', { uuid: vmUuid, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
after(async () => {
|
||||
await cleanupVms(replicatedVmUuids)
|
||||
await teardown(dispatchClient, tracker)
|
||||
})
|
||||
})
|
||||
@@ -36,7 +36,7 @@ describe('Incremental Replication', () => {
|
||||
const setupResult = await setup()
|
||||
dispatchClient = setupResult.dispatchClient
|
||||
tracker = setupResult.tracker
|
||||
vm = setupResult.vm
|
||||
vm = setupResult.vms[0]
|
||||
|
||||
assert.ok(vm, 'Setup should provide a test VM')
|
||||
|
||||
|
||||
@@ -37,8 +37,10 @@ describe('SDN Controller REST API', { skip: !process.env.SDN_CONTROLLER_VM_ID },
|
||||
|
||||
before(async () => {
|
||||
const sdnVmId = process.env.SDN_CONTROLLER_VM_ID
|
||||
|
||||
;({ dispatchClient, tracker, vm } = await setup({ referenceVmId: sdnVmId }))
|
||||
let vms
|
||||
;({ dispatchClient, tracker, vms } = await setup({ referenceVmId: sdnVmId }))
|
||||
// Only using one VM
|
||||
vm = vms[0]
|
||||
|
||||
const vmDetails = await dispatchClient.restApiClient.get(`/rest/v0/vms/${vm.uuid}`)
|
||||
if (!(vmDetails.VIFs?.length > 0)) {
|
||||
|
||||
@@ -35,9 +35,10 @@ async function generateIncrementalVmName(dispatchClient, baseName) {
|
||||
* Test setup with automatic resource creation and tracking.
|
||||
* @param {Object} options - Setup options
|
||||
* @param {string} [options.referenceVmId] - Optional reference VM ID to clone for testing
|
||||
* @param {string} [options.requiredVmQty] - Optional number of clone to create, defaults to 1
|
||||
* @returns {Promise<Object>} Setup result with dispatchClient and created resources
|
||||
*/
|
||||
export const setup = async ({ referenceVmId } = {}) => {
|
||||
export const setup = async ({ referenceVmId, requiredVmQty = 1 } = {}) => {
|
||||
log.debug('Setting up test environment')
|
||||
|
||||
const tracker = createResourceTracker()
|
||||
@@ -45,7 +46,7 @@ export const setup = async ({ referenceVmId } = {}) => {
|
||||
await dispatchClient.initialize()
|
||||
|
||||
const createdResources = {
|
||||
vm: null,
|
||||
vms: [],
|
||||
backupRepository: null,
|
||||
sessionId: tracker.getSessionId(),
|
||||
}
|
||||
@@ -61,21 +62,23 @@ export const setup = async ({ referenceVmId } = {}) => {
|
||||
}
|
||||
|
||||
const vmPrefix = getRequiredEnv('VM_PREFIX')
|
||||
const testVmName = await generateIncrementalVmName(dispatchClient, `${vmPrefix}-QA-Test`)
|
||||
for (let i = 0; i < requiredVmQty; i++) {
|
||||
const testVmName = await generateIncrementalVmName(dispatchClient, `${vmPrefix}-QA-Test`)
|
||||
|
||||
log.debug('Creating test VM', { name: testVmName })
|
||||
log.debug('Creating test VM', { name: testVmName })
|
||||
|
||||
const testVmId = await dispatchClient.vm.clone(referenceVm.uuid, testVmName, {
|
||||
description: `Test VM for QA tests`,
|
||||
fastClone: true,
|
||||
})
|
||||
const testVmId = await dispatchClient.vm.clone(referenceVm.uuid, testVmName, {
|
||||
description: `Test VM for QA tests`,
|
||||
fastClone: true,
|
||||
})
|
||||
|
||||
log.debug('Starting test VM', { id: testVmId })
|
||||
await dispatchClient.vm.start(testVmId)
|
||||
await dispatchClient.vm.waitForPowerState(testVmId, 'Running', 120_000)
|
||||
log.debug('Starting test VM', { id: testVmId })
|
||||
await dispatchClient.vm.start(testVmId)
|
||||
await dispatchClient.vm.waitForPowerState(testVmId, 'Running', 120_000)
|
||||
|
||||
createdResources.vm = await dispatchClient.vm.details(testVmId)
|
||||
tracker.trackResource('vm', testVmId, { name: testVmName, source: referenceVm.name_label })
|
||||
createdResources.vms.push(await dispatchClient.vm.details(testVmId))
|
||||
tracker.trackResource('vm', testVmId, { name: testVmName, source: referenceVm.name_label })
|
||||
}
|
||||
|
||||
// Create or get backup repository
|
||||
const backupRepositoryName = getRequiredEnv('BACKUP_REPOSITORY_NAME')
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { findTaskByMessage } from './index.js'
|
||||
import assert from 'node:assert'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
|
||||
@@ -157,3 +158,52 @@ export const assertBackupSuccess = (result, context = 'Backup') => {
|
||||
assert.strictEqual(result.status, 'success', `${context} should succeed, got '${result.status}': ${details}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that a backup run performed a synchronized snapshot: all VMs were
|
||||
* snapshotted in a single batch phase, before any transfer, and the per-VM
|
||||
* backups reused those batched snapshots (no VM was snapshotted twice).
|
||||
* @param {Object} result - Backup log result
|
||||
* @param {number} vmCount - Number of VMs concerned by this backup job
|
||||
*/
|
||||
export const assertSynchronizedSnapshot = (result, vmCount) => {
|
||||
const snapshotVmTask = findTaskByMessage(result, 'snapshot VMs')
|
||||
assert(snapshotVmTask, `Synchronized backup should have batched snapshots`)
|
||||
assert(
|
||||
snapshotVmTask.tasks && snapshotVmTask.tasks.length === vmCount,
|
||||
`Synchronized backup should have made a snapshot per vm`
|
||||
)
|
||||
|
||||
// Collect every task with a given message across the whole task tree.
|
||||
const collectByMessage = message => {
|
||||
const found = []
|
||||
const walk = tasks => {
|
||||
for (const task of tasks ?? []) {
|
||||
if (task.message === message) {
|
||||
found.push(task)
|
||||
}
|
||||
if (task.tasks) {
|
||||
walk(task.tasks)
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(result.tasks)
|
||||
return found
|
||||
}
|
||||
|
||||
// Every snapshot must come from the batch phase: there should be exactly one
|
||||
// 'snapshot' task per VM (the batched ones) and none taken again during the
|
||||
// per-VM backup. A larger count means a VM was snapshotted a second time at
|
||||
// transfer time, i.e. the backup did NOT reuse the synchronized snapshot.
|
||||
const snapshots = collectByMessage('snapshot')
|
||||
assert(
|
||||
snapshots.length === vmCount,
|
||||
`Synchronized backup should reuse the batched snapshots (expected ${vmCount} 'snapshot' tasks, found ${snapshots.length})`
|
||||
)
|
||||
|
||||
const transfers = collectByMessage('transfer')
|
||||
assert(transfers.length > 0, `Synchronized backup should have transfered snapshots.`)
|
||||
|
||||
const earliestStart = Math.min(...transfers.map(t => t.start))
|
||||
assert(snapshotVmTask.end <= earliestStart, `Synchronized backup snapshots should be complete before transfer starts`)
|
||||
}
|
||||
|
||||
@@ -21,13 +21,12 @@
|
||||
- [XO6/Host] Add possibility to disable a host an evacuate its VMs (PR [#10090](https://github.com/vatesfr/xen-orchestra/pull/10090))
|
||||
- [XO6/Host] Add possibility to reboot a host (PR [#10141](https://github.com/vatesfr/xen-orchestra/pull/10141))
|
||||
- [XO6/Host] Add possibility to force reboot a host (PR [#10175](https://github.com/vatesfr/xen-orchestra/pull/10175))
|
||||
|
||||
- [IPMI-plugin] Add GET plugins/ipmi-sensors/hosts/{id}/ipmi to get IPMI sensors (PR [#10003](https://github.com/vatesfr/xen-orchestra/pull/10003))
|
||||
- [VIF] Add VIF name in header on VIF detail page (PR [#10252](https://github.com/vatesfr/xen-orchestra/pull/10252))
|
||||
|
||||
- [IPMI-plugin] Add GET plugins/ipmi-sensors/hosts/{id}/ipmi to get IPMI sensors (PR [#10003](https://github.com/vatesfr/xen-orchestra/pull/10003))
|
||||
- [VIF] Add VIF name in header on VIF detail page (PR [#10252](https://github.com/vatesfr/xen-orchestra/pull/10252))
|
||||
- [XO6/SR] Add dedicated Storage Repository page with general information, space usage, PBD details, custom fields (PR [#10100](https://github.com/vatesfr/xen-orchestra/pull/10100))
|
||||
- [XO5/Backups] Add `Synchronize snapshots` checkbox to backup jobs to get consistent restore points (PR [#10136](https://github.com/vatesfr/xen-orchestra/pull/10136))
|
||||
|
||||
### Bug fixes
|
||||
|
||||
@@ -61,8 +60,9 @@
|
||||
- @vates/types patch
|
||||
- @xen-orchestra/acl minor
|
||||
- @xen-orchestra/async-map patch
|
||||
- @xen-orchestra/backups patch
|
||||
- @xen-orchestra/backups minor
|
||||
- @xen-orchestra/proxy-cli patch
|
||||
- @xen-orchestra/qa-test minor
|
||||
- @xen-orchestra/rest-api minor
|
||||
- @xen-orchestra/upload-ova patch
|
||||
- @xen-orchestra/web minor
|
||||
@@ -74,6 +74,6 @@
|
||||
- xo-server patch
|
||||
- xo-server-ipmi-sensors minor
|
||||
- xo-server-netbox patch
|
||||
- xo-web patch
|
||||
- xo-web minor
|
||||
|
||||
<!--packages-end-->
|
||||
|
||||
@@ -605,6 +605,9 @@ const messages = {
|
||||
mergeBackupsSynchronously: 'Merge backups synchronously',
|
||||
mergeBackupsSynchronouslyTooltip:
|
||||
'This will use more resources on the backup thread, but ensure there is no locking error when chaining multiple backup jobs on the same remote',
|
||||
synchronizedSnapshot: 'Synchronize snapshots',
|
||||
synchronizedSnapshotTooltip:
|
||||
'Snapshots all VMs together before transfer in order to get consistent restore points. Uses more disks, especially on thick-provisioned SRs',
|
||||
newBackupSelection: 'Select your backup type:',
|
||||
snapshotRetention: 'Snapshot retention',
|
||||
backupName: 'Name',
|
||||
|
||||
@@ -713,6 +713,11 @@ const New = decorate([
|
||||
mergeBackupsSynchronously,
|
||||
})
|
||||
},
|
||||
setSynchronizedSnapshot({ setGlobalSettings }, synchronizedSnapshot) {
|
||||
setGlobalSettings({
|
||||
synchronizedSnapshot,
|
||||
})
|
||||
},
|
||||
setDistributeBackups({ setGlobalSettings }, distributeBackups) {
|
||||
setGlobalSettings({
|
||||
distributeBackups,
|
||||
@@ -741,6 +746,7 @@ const New = decorate([
|
||||
inputNbdConcurrency: generateId,
|
||||
inputNRetriesVmBackupFailures: generateId,
|
||||
inputPreferNbd: generateId,
|
||||
inputSynchronizedSnapshot: generateId,
|
||||
inputDistributeBackups: generateId,
|
||||
inputDistributeReplications: generateId,
|
||||
inputTimeoutId: generateId,
|
||||
@@ -864,6 +870,7 @@ const New = decorate([
|
||||
preferNbd,
|
||||
reportRecipients,
|
||||
reportWhen = 'failure',
|
||||
synchronizedSnapshot,
|
||||
distributeBackups = false,
|
||||
distributeReplications = false,
|
||||
timeout,
|
||||
@@ -1318,6 +1325,21 @@ const New = decorate([
|
||||
onChange={effects.setMergeBackupsSynchronously}
|
||||
/>
|
||||
</FormGroup>
|
||||
<FormGroup>
|
||||
<label htmlFor={state.inputSynchronizedSnapshot}>
|
||||
<strong>{_('synchronizedSnapshot')}</strong>
|
||||
</label>{' '}
|
||||
<Tooltip content={_('synchronizedSnapshotTooltip')}>
|
||||
<Icon icon='info' />
|
||||
</Tooltip>
|
||||
<Toggle
|
||||
className='pull-right'
|
||||
id={state.inputSynchronizedSnapshot}
|
||||
name='synchronizedSnapshot'
|
||||
value={synchronizedSnapshot === true}
|
||||
onChange={effects.setSynchronizedSnapshot}
|
||||
/>
|
||||
</FormGroup>
|
||||
</div>
|
||||
)}
|
||||
</CardBlock>
|
||||
|
||||
Reference in New Issue
Block a user