feat(backup): implement synchronized snapshot option back (#10136)

Adds a synchronizedSnapshot backup setting. When enabled, all of a job's VM snapshots are taken up front in a batch before any transfer begins, and each VM's backup then reuses its pre-taken snapshot instead of snapshotting again.

Settings:
synchronizedSnapshot:

    false (default)
    true (synchronize every VM in the job)
    <tag> (synchronize only VMs carrying that tag, others are backed up normally).

snapshotConcurrency: max concurrent snapshots during the batch phase (default 2).

A batch member that fails to open is silently dropped from synchronization and backed up normally.

Unit tested where applicable and e2e tests added in qa-test.

Checkbox added in xo5 backup jobs to synchronize snapshots, tags not supported yet.
This commit is contained in:
spacotte-vates
2026-08-24 17:58:03 +02:00
committed by GitHub
parent 814bc96a8b
commit f812eaaeb6
21 changed files with 913 additions and 53 deletions

View File

@@ -9,7 +9,9 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
import { FullXapi } from './_vmRunners/FullXapi.mjs'
import { Throttle } from '@vates/generator-toolbox'
import { asyncEach } from '@vates/async-each'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
const noop = Function.prototype
@@ -31,7 +33,9 @@ const DEFAULT_XAPI_VM_SETTINGS = {
nRetriesVmBackupFailures: 0,
offlineBackup: false,
offlineSnapshot: false,
snapshotConcurrency: 2,
snapshotRetention: 0,
synchronizedSnapshot: false,
timeout: 0,
useNbd: false,
unconditionalSnapshot: false,
@@ -47,6 +51,16 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
return baseSettings
}
_getVmBackup(jobMode, opts) {
if (jobMode === 'delta') {
return new IncrementalXapi(opts)
} else if (jobMode === 'full') {
return new FullXapi(opts)
}
throw new Error(`Job mode ${jobMode} not implemented`)
}
async run() {
const job = this._job
@@ -95,7 +109,81 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmIds)
const preTakenTimestampByVmId = {}
const failedSnapshotByVmId = {}
if (settings.synchronizedSnapshot) {
await Task.run({ properties: { name: 'snapshot VMs' } }, async () => {
await Disposable.use(
Disposable.all(vmIds.map(vmId => this._getRecord('VM', vmId).catch(noop))),
async vms => {
// remove vms that failed (already handled)
vms = vms.filter(_ => _ !== undefined)
const batchIds = selectSynchronizedSnapshotVms(settings.synchronizedSnapshot, vms)
await asyncEach(
[...vms].filter(vm => batchIds.has(vm.uuid)),
async vm => {
const vmSettings = { ...settings, ...allSettings[vm.uuid] }
const opts = {
baseSettings,
config,
getSnapshotNameLabel,
healthCheckSr,
job,
remoteAdapters,
schedule,
settings: vmSettings,
srs,
throttleGenerator,
throttleStream,
vm,
}
let vmBackup
try {
vmBackup = this._getVmBackup(job.mode, opts)
if (
!vmBackup._settings.offlineBackup &&
!vmBackup._settings.offlineSnapshot &&
(await vmBackup._mustDoSnapshot())
) {
await vmBackup._prepareAndSnapshot()
preTakenTimestampByVmId[vm.uuid] = vmBackup.timestamp
}
} catch (error) {
failedSnapshotByVmId[vm.uuid] = error
if (vmBackup !== undefined) {
try {
await vmBackup._fetchJobSnapshots()
await vmBackup._removeUnusedSnapshots()
await vmBackup._cleanMetadata()
} catch (cleanupError) {
// Best effort cleanup
}
}
}
},
{
concurrency: settings.snapshotConcurrency,
}
)
}
)
})
}
const snapshotedVmIds = new Set(vmIds.filter(id => !(id in failedSnapshotByVmId)))
Object.entries(failedSnapshotByVmId).forEach(([vmId, error]) => {
Task.run(
{
properties: { id: vmId, name: 'backup VM', type: 'VM' },
},
() => Promise.reject(error)
).catch(noop)
})
const queue = new Set(snapshotedVmIds)
const taskByVmId = {}
const nTriesByVmId = {}
@@ -155,21 +243,16 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
schedule,
settings: vmSettings,
srs,
// when set, the batch phase already snapshotted this VM; the
// runner re-finds that snapshot by its metadata (see _snapshot)
synchronizedSnapshotTimestamp: preTakenTimestampByVmId[vmUuid],
throttleGenerator,
throttleStream,
vm,
}
let vmBackup
if (job.mode === 'delta') {
vmBackup = new IncrementalXapi(opts)
} else {
if (job.mode === 'full') {
vmBackup = new FullXapi(opts)
} else {
throw new Error(`Job mode ${job.mode} not implemented`)
}
}
const vmBackup = this._getVmBackup(job.mode, opts)
return vmBackup.run().catch(error => {
taskError = error
})

View File

@@ -0,0 +1,19 @@
import { Task } from '@vates/task'
export function selectSynchronizedSnapshotVms(synchronizedSnapshot, vms) {
if (synchronizedSnapshot === false) {
return new Set()
}
let matches
if (typeof synchronizedSnapshot === 'string') {
matches = vms.filter(vm => vm.tags.includes(synchronizedSnapshot))
} else if (synchronizedSnapshot === true) {
matches = vms
} else {
Task.warning('unsupported synchronizedSnapshot option, expected a boolean or a tag name', { synchronizedSnapshot })
return new Set()
}
return new Set(matches.map(vm => vm.uuid))
}

View File

@@ -0,0 +1,73 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
// Helper mirroring the minimal VM record shape the selector needs.
const vm = (uuid, tags = []) => ({ uuid, tags })
// `selectSynchronizedSnapshotVms(synchronizedSnapshot, vms)` must return a Set
// of the UUIDs that should be snapshotted together in the batch phase.
// - false -> feature off, no VM is batched
// - true -> every VM is batched, regardless of tags
// - '<tagName>' -> only VMs carrying that exact tag are batched
const tests = [
{
label: 'false disables batching',
setting: false,
vms: [vm('a', ['prod']), vm('b')],
expected: [],
},
{
label: 'true batches every VM regardless of tags',
setting: true,
vms: [vm('a', ['prod']), vm('b')],
expected: ['a', 'b'],
},
{
label: 'tag batches only the VMs carrying it',
setting: 'prod',
vms: [vm('a', ['prod']), vm('b', ['dev']), vm('c', ['prod', 'other'])],
expected: ['a', 'c'],
},
{
label: 'tag carried by no VM yields an empty batch',
setting: 'prod',
vms: [vm('a', ['dev']), vm('b')],
expected: [],
},
{
label: 'tag carried by every VM batches all of them',
setting: 'prod',
vms: [vm('a', ['prod']), vm('b', ['prod'])],
expected: ['a', 'b'],
},
{
label: 'empty VM list yields an empty batch',
setting: true,
vms: [],
expected: [],
},
{
label: 'empty-string tag matches no VM when none carries an empty tag',
setting: '',
vms: [vm('a', ['prod'])],
expected: [],
},
{
label: 'non boolean or string yiels an empty batch',
setting: 1,
vms: [vm('a', ['prod'])],
expected: [],
},
]
describe('selectSynchronizedSnapshotVms()', () => {
for (const { label, setting, vms, expected } of tests) {
it(label, () => {
const result = selectSynchronizedSnapshotVms(setting, vms)
assert.ok(result instanceof Set, 'result must be a Set')
assert.deepEqual([...result].sort(), [...expected].sort())
})
}
})

View File

@@ -80,14 +80,14 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
const sizeContainer = watchStreamSize(stream)
const timestamp = Date.now()
const transferStart = Date.now()
await this._callWriters(
writer =>
writer.run({
maxStreamLength,
sizeContainer,
stream: forkStreamUnpipe(stream),
timestamp,
timestamp: this.timestamp,
vm,
vmSnapshot: exportedVm,
}),
@@ -95,8 +95,7 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
)
const { size } = sizeContainer
const end = Date.now()
const duration = end - timestamp
const duration = Date.now() - transferStart
debug('transfer complete', {
duration,
speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,

View File

@@ -13,7 +13,6 @@ import {
DELTA_CHAIN_LENGTH,
EXPORTED_SUCCESSFULLY,
setVmDeltaChainLength,
markExportSuccessfull,
} from '../../_otherConfig.mjs'
import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
@@ -68,7 +67,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
// @todo : reimplement throttle
const timestamp = Date.now()
await this._callWriters(
writer =>
writer.transfer({
@@ -78,7 +76,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
// clean; the tip's flag lets the next run skip re-probing
includeNonNbdQcow2Fix: true,
isVhdDifferencing,
timestamp,
timestamp: this.timestamp,
vm,
vmSnapshot: exportedVm,
}),
@@ -91,11 +89,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
}
// not the case if offlineBackup
if (exportedVm.is_a_snapshot) {
await markExportSuccessfull(this._xapi, exportedVm.$ref)
}
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
}

View File

@@ -7,6 +7,7 @@ import { asyncEach } from '@vates/async-each'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { Task } from '@vates/task'
import { formatDateTime } from '@xen-orchestra/xapi'
import { getOldEntries } from '../../_getOldEntries.mjs'
import { Abstract } from './_Abstract.mjs'
@@ -16,9 +17,11 @@ import {
JOB_ID,
SCHEDULE_ID,
VM_UUID,
EXPORTED_SUCCESSFULLY,
resetVmOtherConfig,
setVmOtherConfig,
setVmSnapshotContentKeys,
markExportSuccessfull,
} from '../../_otherConfig.mjs'
const { warn, info } = createLogger('xo:backups:AbstractXapi')
@@ -36,6 +39,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
schedule,
settings,
srs,
synchronizedSnapshotTimestamp,
throttleGenerator,
throttleStream,
vm,
@@ -63,9 +67,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
// VM (snapshot) that is really exported
this._exportedVm = undefined
this._synchronizedSnapshotTimestamp = synchronizedSnapshotTimestamp
this._vm = vm
this._baseVdis = undefined
this._getSnapshotNameLabel = getSnapshotNameLabel
this._isIncremental = job.mode === 'delta'
this._healthCheckSr = healthCheckSr
@@ -180,6 +184,27 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
}
async _snapshot() {
if (this._synchronizedSnapshotTimestamp !== undefined) {
const datetime = formatDateTime(this._synchronizedSnapshotTimestamp)
const candidates = this._vm.$snapshots.filter(
snapshot =>
snapshot?.other_config[JOB_ID] === this._jobId &&
snapshot?.other_config[DATETIME] === datetime &&
!snapshot.other_config[EXPORTED_SUCCESSFULLY]
)
if (candidates.length === 1) {
this._exportedVm = candidates[0]
this.timestamp = this._synchronizedSnapshotTimestamp
return
}
warn('expected exactly one synchronized snapshot to reuse, taking a fresh one', {
vm: this._vm.uuid,
datetime,
count: candidates.length,
})
}
const vm = this._vm
const xapi = this._xapi
@@ -396,7 +421,20 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await xapi.barrier()
// ensure cached object are up to date
this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
let disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
// The synchronized snapshot for this run is taken up-front by the batch
// phase but transferred later. Until it has been exported, hide it from
// retention so it is neither reclaimed nor allowed to steal the delta
// base's "most recent" protection. This makes the pre-transfer pass behave
// like a normal run, where the snapshot does not exist yet.
if (this._synchronizedSnapshotTimestamp !== undefined) {
const datetime = formatDateTime(this._synchronizedSnapshotTimestamp)
const isInFlightSyncSnapshot = ({ other_config }) =>
other_config[DATETIME] === datetime && !other_config[EXPORTED_SUCCESSFULLY]
this._jobSnapshotVdis = this._jobSnapshotVdis.filter(vdi => !isInFlightSyncSnapshot(vdi))
disklessVmSnapshots = disklessVmSnapshots.filter(vm => !isInFlightSyncSnapshot(vm))
}
// get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
@@ -464,6 +502,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
vm = vdiVm
}
}
if (vm?.$ref !== undefined) {
return xapi.VM_destroy(vm.$ref)
} else {
@@ -493,6 +532,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
return
}
await xapi.VM_destroy(snapshotPerDatetime[datetime])
})
})
@@ -551,6 +591,11 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
throw new Error('Not implemented')
}
async _prepareAndSnapshot() {
await this._cleanMetadata()
await this._snapshot()
}
async run($defer) {
const settings = this._settings
assert(
@@ -614,6 +659,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
if (this._writers.size !== 0) {
await this._copy()
}
// not the case if offlineBackup
if (this._exportedVm.is_a_snapshot) {
await markExportSuccessfull(this._xapi, this._exportedVm.$ref)
}
} finally {
if (startAfter) {
ignoreErrors.call(vm.$callAsync('start', false, false))

View File

@@ -0,0 +1,327 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { formatDateTime } from '@xen-orchestra/xapi'
import { AbstractXapi } from './_AbstractXapi.mjs'
import { DATETIME, EXPORTED_SUCCESSFULLY, JOB_ID, SCHEDULE_ID } from '../../_otherConfig.mjs'
// Build an AbstractXapi instance without running its heavy constructor, then
// assign only the fields the method under test reads.
const makeRunner = props => Object.assign(Object.create(AbstractXapi.prototype), props)
describe('_snapshot() synchronized-snapshot reuse guard', () => {
it('reuses the batch snapshot, found by metadata, without snapshotting again', async () => {
let mustDoSnapshotCalls = 0
const timestamp = 1717200000000
// the snapshot taken by the batch phase, identified by its other_config
const batchSnapshot = {
$ref: 'batch-snapshot-ref',
other_config: {
[JOB_ID]: 'job-1',
[SCHEDULE_ID]: 'schedule-1',
[DATETIME]: formatDateTime(timestamp),
},
}
const runner = makeRunner({
_synchronizedSnapshotTimestamp: timestamp,
_jobId: 'job-1',
scheduleId: 'schedule-1',
_vm: { uuid: 'vm-uuid', $snapshots: [batchSnapshot] },
_xapi: { barrier: async () => {} },
// if the metadata lookup fails, _snapshot() falls through to here
_mustDoSnapshot: async () => {
mustDoSnapshotCalls++
return false
},
})
await runner._snapshot()
assert.equal(mustDoSnapshotCalls, 0, '_mustDoSnapshot() should not be called when the batch snapshot is found')
assert.equal(runner._exportedVm, batchSnapshot, '_exportedVm should be the batch snapshot found by metadata')
assert.equal(runner.timestamp, timestamp, 'timestamp should be the synchronized snapshot timestamp')
})
it('takes a fresh snapshot when the batch snapshot can no longer be found', async () => {
let mustDoSnapshotCalls = 0
const vm = { uuid: 'vm-uuid', $snapshots: [] } // the pre-taken snapshot is gone
const runner = makeRunner({
_synchronizedSnapshotTimestamp: 1717200000000,
_jobId: 'job-1',
scheduleId: 'schedule-1',
_vm: vm,
_xapi: { barrier: async () => {} },
_settings: {},
_mustDoSnapshot: async () => {
mustDoSnapshotCalls++
return false
},
})
await runner._snapshot()
assert.equal(
mustDoSnapshotCalls,
1,
'a missing batch snapshot should not be reused: _snapshot() should fall through and take a fresh snapshot'
)
assert.equal(
runner._exportedVm,
vm,
'_exportedVm should fall back to the freshly-snapshotted VM, not stay on a stale/missing batch snapshot'
)
})
it('without a pre-taken snapshot and no snapshot needed, exports the live VM', async () => {
let mustDoSnapshotCalls = 0
const vm = { uuid: 'vm-uuid' }
const runner = makeRunner({
_exportedVm: undefined,
_vm: vm,
_xapi: {},
_settings: {},
_mustDoSnapshot: async () => {
mustDoSnapshotCalls++
return false
},
})
await runner._snapshot()
assert.equal(mustDoSnapshotCalls, 1, '_mustDoSnapshot() should be called once when no snapshot was pre-taken')
assert.equal(runner._exportedVm, vm, '_exportedVm should be set to the live VM when no snapshot is needed')
assert.equal(typeof runner.timestamp, 'number', 'timestamp should be set when exporting the live VM')
})
})
describe('_removeUnusedSnapshots() protects the pre-taken synchronized snapshot', () => {
const OLD_DATETIME = '20240101T00:00:00Z'
const SYNC_TIMESTAMP = 1717200000000
const SYNC_DATETIME = formatDateTime(SYNC_TIMESTAMP)
// Full-mode job with snapshotRetention 0, so retention wants to remove every
// snapshot. `vdi-fresh` is the snapshot taken up-front by the synchronized
// batch phase (identified by SYNC_TIMESTAMP); until it is exported it must be
// hidden from retention.
const makeRemoveRunner = ({ exported }) => {
const destroyed = []
const snapshotVm = ($ref, name_label) => ({
$ref,
name_label,
is_control_domain: false,
$snapshot_of: 'live-vm-ref',
other_config: {},
})
const freshSnapshotVm = snapshotVm('vm-fresh', 'fresh')
const oldSnapshotVm = snapshotVm('vm-old', 'old')
const vdi = ($ref, datetime, snapshotVmRecord, isExported = false) => ({
$ref,
other_config: {
[DATETIME]: datetime,
[SCHEDULE_ID]: 'schedule-1',
...(isExported ? { [EXPORTED_SUCCESSFULLY]: 'true' } : {}),
},
$VBDs: [{ $VM: snapshotVmRecord }],
})
// the old snapshot was exported by a previous run
const oldVdi = vdi('vdi-old', OLD_DATETIME, oldSnapshotVm, true)
const freshVdi = vdi('vdi-fresh', SYNC_DATETIME, freshSnapshotVm, exported)
const registry = { 'vdi-old': oldVdi, 'vdi-fresh': freshVdi }
const runner = makeRunner({
_synchronizedSnapshotTimestamp: SYNC_TIMESTAMP,
_vm: { uuid: 'live-uuid', $snapshots: [] },
_baseSettings: { snapshotRetention: 0 },
_jobSnapshotVdis: [oldVdi, freshVdi],
_disklessJobSnapshotVms: [],
job: { mode: 'full', settings: {} },
_xapi: {
barrier: async () => {},
getObject: ref => registry[ref],
VM_destroy: async ref => {
destroyed.push(ref)
},
VDI_destroy: async ref => {
destroyed.push(ref)
},
},
})
return { runner, destroyed }
}
it('does not destroy the pre-taken snapshot before it has been transferred', async () => {
const { runner, destroyed } = makeRemoveRunner({ exported: false })
await runner._removeUnusedSnapshots()
assert.deepEqual(destroyed, ['vm-old'], 'only the older snapshot should be removed')
})
it('destroys the snapshot once it has been transferred (retention 0, no leak)', async () => {
const { runner, destroyed } = makeRemoveRunner({ exported: true })
await runner._removeUnusedSnapshots()
assert.deepEqual(destroyed.sort(), ['vm-fresh', 'vm-old'], 'both snapshots should be removed after transfer')
})
})
describe('_removeUnusedSnapshots() in the synchronized batch pre-snapshot state', () => {
it('delta mode: keeps the base snapshot when the fresh one has not been taken yet', async () => {
const destroyed = []
const BASE_DATETIME = '2024-06-01T00:00:00Z'
const baseSnapshotVm = {
$ref: 'vm-base',
name_label: 'base',
is_control_domain: false,
$snapshot_of: 'live-vm-ref',
// exported successfully by the previous run
other_config: { [EXPORTED_SUCCESSFULLY]: 'true' },
}
const baseVdi = {
$ref: 'vdi-base',
other_config: { [DATETIME]: BASE_DATETIME, [SCHEDULE_ID]: 'schedule-1' },
$VBDs: [{ $VM: baseSnapshotVm }],
}
const registry = { 'vdi-base': baseVdi }
const runner = makeRunner({
// batch pre-snapshot state: the synchronized snapshot has not been taken yet
_exportedVm: undefined,
_vm: { uuid: 'live-uuid', $snapshots: [] },
_baseSettings: { snapshotRetention: 0 },
_jobSnapshotVdis: [baseVdi],
_disklessJobSnapshotVms: [],
job: { mode: 'delta', settings: {} },
_xapi: {
barrier: async () => {},
getObject: ref => registry[ref],
VM_destroy: async ref => {
destroyed.push(ref)
},
VDI_destroy: async ref => {
destroyed.push(ref)
},
},
})
await runner._removeUnusedSnapshots()
assert.deepEqual(destroyed, [], 'the base snapshot must be kept as the delta base for the upcoming transfer')
})
})
describe('_removeUnusedSnapshots() reclaims orphan / CBT snapshot VDIs (no attached VM)', () => {
// Guards the `else` branch: snapshot VDIs that are not attached to any user VM
// (e.g. CBT metadata, orphans) must be reclaimed via VDI_destroy, not VM_destroy.
it('destroys snapshot VDIs directly when they are not attached to any VM', async () => {
const vmDestroyed = []
const vdiDestroyed = []
const orphanVdi = {
$ref: 'vdi-orphan',
other_config: { [DATETIME]: '2024-01-01T00:00:00Z', [SCHEDULE_ID]: 'schedule-1' },
$VBDs: [], // not attached to any VM
}
const registry = { 'vdi-orphan': orphanVdi }
const runner = makeRunner({
_exportedVm: undefined,
_vm: { uuid: 'live-uuid', $snapshots: [] },
_baseSettings: { snapshotRetention: 0 },
_jobSnapshotVdis: [orphanVdi],
_disklessJobSnapshotVms: [],
job: { mode: 'full', settings: {} },
_xapi: {
barrier: async () => {},
getObject: ref => registry[ref],
VM_destroy: async ref => {
vmDestroyed.push(ref)
},
VDI_destroy: async ref => {
vdiDestroyed.push(ref)
},
},
})
await runner._removeUnusedSnapshots()
assert.deepEqual(vdiDestroyed, ['vdi-orphan'], 'the orphan VDI should be reclaimed via VDI_destroy')
assert.deepEqual(vmDestroyed, [], 'no VM should be destroyed for an orphan VDI')
})
})
describe('_removeUnusedSnapshots() diskless VM snapshots', () => {
const OLD_DATETIME = '20240101T00:00:00Z'
const SYNC_TIMESTAMP = 1717200000000
const SYNC_DATETIME = formatDateTime(SYNC_TIMESTAMP)
// A diskless VM's backup snapshots are tracked as VM snapshots (no VDIs to
// anchor them). `dl-fresh` is the snapshot the synchronized batch pre-took,
// identified by SYNC_TIMESTAMP.
const makeDisklessRunner = ({ exported, mode = 'full' }) => {
const destroyed = []
const oldSnap = { $ref: 'dl-old', other_config: { [DATETIME]: OLD_DATETIME, [SCHEDULE_ID]: 'schedule-1' } }
const freshSnap = { $ref: 'dl-fresh', other_config: { [DATETIME]: SYNC_DATETIME, [SCHEDULE_ID]: 'schedule-1' } }
if (exported) freshSnap.other_config[EXPORTED_SUCCESSFULLY] = 'true'
const registry = { 'dl-old': oldSnap, 'dl-fresh': freshSnap }
const runner = makeRunner({
_synchronizedSnapshotTimestamp: SYNC_TIMESTAMP,
_vm: { uuid: 'live-uuid', $snapshots: [] },
_baseSettings: { snapshotRetention: 0 },
_jobSnapshotVdis: [],
_disklessJobSnapshotVms: [oldSnap, freshSnap],
job: { mode, settings: {} },
_xapi: {
barrier: async () => {},
getObject: ref => registry[ref],
VM_destroy: async ref => {
destroyed.push(ref)
},
VDI_destroy: async ref => {
destroyed.push(ref)
},
},
})
return { runner, destroyed }
}
it('does not destroy the pre-taken diskless snapshot before it has been transferred', async () => {
const { runner, destroyed } = makeDisklessRunner({ exported: false })
await runner._removeUnusedSnapshots()
assert.deepEqual(destroyed, ['dl-old'], 'only the older diskless snapshot should be removed')
})
it('destroys the diskless snapshot once it has been transferred (retention 0, no leak)', async () => {
const { runner, destroyed } = makeDisklessRunner({ exported: true })
await runner._removeUnusedSnapshots()
assert.deepEqual(
destroyed.sort(),
['dl-fresh', 'dl-old'],
'both diskless snapshots should be removed after transfer'
)
})
it('keeps the most recent diskless snapshot in delta mode (base for next delta)', async () => {
// Even after transfer (exported), delta mode must retain the latest
// snapshot so the next run can compute its delta against it.
const { runner, destroyed } = makeDisklessRunner({ exported: true, mode: 'delta' })
await runner._removeUnusedSnapshots()
assert.deepEqual(destroyed, ['dl-old'], 'the most recent diskless snapshot should be kept in delta mode')
})
})

View File

@@ -11,7 +11,7 @@
* @param {number} [options.nbdConcurrency] - Number of concurrent NBD connections (default: 1)
* @returns {Object} Complete backup job configuration
*/
export function backupConfig(name, schedule, vm, backupRepository, options = {}) {
export function backupConfig(name, schedule, vms, backupRepository, options = {}) {
const config = {
name,
mode: 'delta', // Will be overridden by tests
@@ -26,9 +26,7 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {})
bypassVdiChainsCheck: true,
},
},
vms: {
[vm.uuid]: vm,
},
vms: Array.isArray(vms) ? Object.fromEntries(vms.map(vm => [vm.uuid, vm])) : { [vms.uuid]: vms },
remotes: {
[backupRepository.id]: backupRepository,
},
@@ -48,6 +46,14 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {})
config.settings[''].nbdConcurrency = options.nbdConcurrency
}
if (options.snapshotConcurrency !== undefined) {
config.settings[''].snapshotConcurrency = options.snapshotConcurrency
}
if (options.synchronizedSnapshot !== undefined) {
config.settings[''].synchronizedSnapshot = options.synchronizedSnapshot
}
return config
}

View File

@@ -37,6 +37,7 @@
"qa:backup:nbd": "node --env-file-if-exists=.env --test tests/backup.nbd.test.js",
"qa:backup:combined": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js",
"qa:backup:combined:cycle": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='Full Cycle'",
"qa:backup:sync": "node --env-file-if-exists=.env --test tests/backup.synchronized.test.js --test-concurrency=1",
"qa:mirror:cr": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='CR Mode'",
"qa:mirror": "node --env-file-if-exists=.env --test tests/backup-mirror.test.js",
"qa:replication": "node --env-file-if-exists=.env --test --test-concurrency=1 tests/replication.test.js",

View File

@@ -143,7 +143,8 @@ describe('Mirror Backup - Full Remote', () => {
const setupResult = await setup()
dispatchClient = setupResult.dispatchClient
tracker = setupResult.tracker
vm = setupResult.vm
// Only using one VM
vm = setupResult.vms[0]
assert.ok(vm, 'Setup should provide a test VM')
log.debug('Using test VM', { name: vm.name_label, uuid: vm.uuid })

View File

@@ -108,7 +108,8 @@ describe('Backup + Replication Combined Tests', () => {
const setupResult = await setup()
dispatchClient = setupResult.dispatchClient
tracker = setupResult.tracker
vm = setupResult.vm
// Only using one VM
vm = setupResult.vms[0]
assert.ok(vm, 'Setup should provide a test VM')
log.debug('Using test VM from setup', { name: vm.name_label, uuid: vm.uuid })

View File

@@ -23,7 +23,10 @@ describe('Incremental backup file restore', () => {
let incrementalBackup
before(async () => {
;({ dispatchClient, tracker, vm, backupRepository } = await setup())
let vms
;({ dispatchClient, tracker, vms, backupRepository } = await setup())
// Only using one VM
vm = vms[0]
const name = generateBackupJobName()
const schedule = getDefaultSchedule()

View File

@@ -22,6 +22,7 @@ const log = createLogger('qa:backup:nbd')
describe('NBD Incremental Backup Tests', () => {
let vm
let vms
let backupRepository
let healthCheckSr
let backupJob
@@ -31,7 +32,9 @@ describe('NBD Incremental Backup Tests', () => {
let createBackupJobForTest
before(async () => {
;({ dispatchClient, tracker } = await setup())
;({ dispatchClient, tracker, vms } = await setup())
// Only using one VM
vm = vms[0]
// Look for test VMs with incremental naming pattern
const vmPrefix = getRequiredEnv('VM_PREFIX')

View File

@@ -0,0 +1,222 @@
import assert from 'node:assert'
import { after, before, describe, it } from 'node:test'
import { createLogger } from '@xen-orchestra/log'
import { backupConfig } from '../backup.config.js'
import { FilterBuilder } from '../client/FilterBuilder.js'
import {
assertFullOrDelta,
assertFullOrDeltaForSr,
findTaskByMessage,
generateBackupJobName,
getDefaultSchedule,
getScheduleKey,
getRequiredEnv,
} from '../utils/index.js'
import { assertBackupSuccess, assertSynchronizedSnapshot } from '../utils/backupUtils.js'
import { setup, teardown } from './setup.js'
const log = createLogger('qa:backup:base')
describe('Backup basic tests', () => {
let vms
let backupRepository
let healthCheckSr
let name
let defaultSchedule
let backupJob
let backupJobId
let defaultConfig
let synchronizedConfig
let dispatchClient
let tracker
let createBackupJobForTest
// Replica VMs created on the destination SR by the CR test; the resource
// tracker only tracks the job, so they must be deleted explicitly.
const replicatedVmUuids = []
before(async () => {
;({ dispatchClient, tracker } = await setup({ requiredVmQty: 2 }))
// Look for test VMs with incremental naming pattern
const vmPrefix = getRequiredEnv('VM_PREFIX')
const filter = FilterBuilder.create().withGlob('name_label', `${vmPrefix}-QA-Test-*`)
const qaVms = await dispatchClient.vm.list(filter)
assert(
qaVms.length >= 2,
`at least 2 VM with pattern "${vmPrefix}-QA-Test-*" are required - backup tests cannot run`
)
// Use the first available QA VM for backup tests
vms = qaVms.slice(0, 2)
log.debug(
'Found test VMs for backup sync tests',
{ name: vms[0].name_label, uuid: vms[0].uuid },
{ name: vms[1].name_label, uuid: vms[1].uuid }
)
const backupRepositoryName = getRequiredEnv('BACKUP_REPOSITORY_NAME')
backupRepository = await dispatchClient.backupRepository.get({ name: backupRepositoryName })
if (!backupRepository) {
log.warn('Backup repository not found, creating it for tests', { name: backupRepositoryName })
// Create the backup repository for testing
try {
const backupRepositoryId = await dispatchClient.backupRepository.create(backupRepositoryName, {
url: getRequiredEnv('BACKUP_REPOSITORY_URL'),
})
// Fetch the canonical repository object from the API
// eslint-disable-next-line require-atomic-updates -- sequential code in before() hook, no race condition
backupRepository = await dispatchClient.backupRepository.get({ id: backupRepositoryId })
if (!backupRepository) {
throw new Error(`Failed to retrieve created backup repository ${backupRepositoryId}`)
}
// Track the newly created repository for cleanup
tracker.trackResource('backupRepository', backupRepositoryId, { name: backupRepositoryName })
} catch (error) {
log.warn('Failed to create test backup repository', { error })
assert.fail(
`Backup repository "${backupRepositoryName}" is required for backup tests - could not create it: ${error.message}`
)
}
}
// Get SR for health checks by ID
const srId = getRequiredEnv('SR_ID')
log.debug('Getting SR for health checks', { srId })
healthCheckSr = await dispatchClient.sr.details(srId)
if (!healthCheckSr) {
throw new Error(`SR with ID "${srId}" not found - cannot run backup tests with health checks`)
}
log.debug('Found SR for health checks', { name: healthCheckSr.name_label })
name = generateBackupJobName()
defaultSchedule = getDefaultSchedule()
defaultConfig = backupConfig(name, defaultSchedule, vms, backupRepository)
synchronizedConfig = backupConfig(name, defaultSchedule, vms, backupRepository, {
synchronizedSnapshot: true,
})
// Functions for easy tests
createBackupJobForTest = async (config, mode) => {
config.mode = mode
backupJobId = await dispatchClient.backup.createBackupJob(config)
backupJob = await dispatchClient.backup.details(backupJobId)
assert(backupJob.mode === mode)
// Track the backup job and its schedule for cleanup
tracker.trackResource('backupJob', backupJobId, { name: config.name, mode })
// Track the schedule (schedules are in the settings object)
const scheduleKey = getScheduleKey(backupJob)
if (scheduleKey) {
tracker.trackResource('schedule', scheduleKey, { name: config.name, backupJobId })
}
}
})
describe('Run synchronized backup jobs', () => {
it('should run the backup job in full mode', async () => {
await createBackupJobForTest(defaultConfig, 'full')
const job = await dispatchClient.backup.details(backupJobId)
const realScheduleKey = getScheduleKey(job)
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
assertBackupSuccess(result, 'Synchronized full backup')
assertFullOrDelta(result, backupRepository.id, { mustBeFull: true })
assert(findTaskByMessage(result, 'snapshot VMs') === null, `No batch snapshot should have been performed`)
})
it('should run a synchronized delta backup and reuse the batch snapshots as the delta base on the second run', async () => {
await createBackupJobForTest(synchronizedConfig, 'delta')
const job = await dispatchClient.backup.details(backupJobId)
const realScheduleKey = getScheduleKey(job)
// Does 2 backups, first one is full synchronized, second one is a delta.y
for (let index = 0; index < 2; index++) {
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
assertBackupSuccess(result, index === 0 ? 'Synchronized full backup' : 'Synchronized delta backup')
assertFullOrDelta(result, backupRepository.id, { mustBeFull: index === 0 })
assertSynchronizedSnapshot(result, vms.length)
}
})
it('should replicate a synchronized delta backup and reuse the batch snapshots on the second run', async () => {
// reuse the same SR the suite already requires; same-SR replication is fine
const targetSrId = getRequiredEnv('SR_ID')
const crConfig = {
name: generateBackupJobName(),
mode: 'delta',
schedules: { '': getDefaultSchedule() },
settings: {
'': {
timezone: 'Europe/Paris',
copyRetention: 3,
preferNbd: true,
bypassVdiChainsCheck: true,
synchronizedSnapshot: true,
},
},
vms: Object.fromEntries(vms.map(vm => [vm.uuid, vm])),
srs: { [targetSrId]: true },
}
backupJobId = await dispatchClient.backup.createBackupJob(crConfig)
const job = await dispatchClient.backup.details(backupJobId)
assert(job.mode === 'delta')
tracker.trackResource('backupJob', backupJobId, { name: crConfig.name, mode: 'delta' })
const realScheduleKey = getScheduleKey(job)
if (realScheduleKey) {
tracker.trackResource('schedule', realScheduleKey, { name: crConfig.name, backupJobId })
}
const vmUuidsBefore = new Set((await dispatchClient.vm.list()).map(vm => vm.uuid))
// First run is a full replication (creates the replicas); the second must
// come back as an incremental, which is only possible if the batch
// snapshots survived the pre-transfer retention pass as the delta base.
for (let index = 0; index < 2; index++) {
const result = await dispatchClient.backup.runJobAndGetLog(backupJobId, realScheduleKey)
assertBackupSuccess(result, index === 0 ? 'Synchronized full replication' : 'Synchronized delta replication')
assertFullOrDeltaForSr(result, targetSrId, { mustBeFull: index === 0 })
assertSynchronizedSnapshot(result, vms.length)
}
const replicas = (await dispatchClient.vm.list()).filter(vm => !vmUuidsBefore.has(vm.uuid)).map(vm => vm.uuid)
replicatedVmUuids.push(...replicas)
assert.strictEqual(replicas.length, vms.length, 'one replica should be created per source VM')
})
})
const cleanupVms = async vmUuids => {
for (const vmUuid of vmUuids) {
try {
const vmDetails = await dispatchClient.vm.details(vmUuid)
if (vmDetails?.power_state === 'Running') {
await dispatchClient.vm.stop(vmUuid, { force: true })
await dispatchClient.vm.waitForPowerState(vmUuid, 'Halted', 60_000)
}
await dispatchClient.vm.delete(vmUuid, { deleteDisks: true })
log.debug('Cleaned up replica VM', { uuid: vmUuid })
} catch (error) {
log.warn('Failed to clean up replica VM', { uuid: vmUuid, error })
}
}
}
after(async () => {
await cleanupVms(replicatedVmUuids)
await teardown(dispatchClient, tracker)
})
})

View File

@@ -36,7 +36,7 @@ describe('Incremental Replication', () => {
const setupResult = await setup()
dispatchClient = setupResult.dispatchClient
tracker = setupResult.tracker
vm = setupResult.vm
vm = setupResult.vms[0]
assert.ok(vm, 'Setup should provide a test VM')

View File

@@ -37,8 +37,10 @@ describe('SDN Controller REST API', { skip: !process.env.SDN_CONTROLLER_VM_ID },
before(async () => {
const sdnVmId = process.env.SDN_CONTROLLER_VM_ID
;({ dispatchClient, tracker, vm } = await setup({ referenceVmId: sdnVmId }))
let vms
;({ dispatchClient, tracker, vms } = await setup({ referenceVmId: sdnVmId }))
// Only using one VM
vm = vms[0]
const vmDetails = await dispatchClient.restApiClient.get(`/rest/v0/vms/${vm.uuid}`)
if (!(vmDetails.VIFs?.length > 0)) {

View File

@@ -35,9 +35,10 @@ async function generateIncrementalVmName(dispatchClient, baseName) {
* Test setup with automatic resource creation and tracking.
* @param {Object} options - Setup options
* @param {string} [options.referenceVmId] - Optional reference VM ID to clone for testing
* @param {string} [options.requiredVmQty] - Optional number of clone to create, defaults to 1
* @returns {Promise<Object>} Setup result with dispatchClient and created resources
*/
export const setup = async ({ referenceVmId } = {}) => {
export const setup = async ({ referenceVmId, requiredVmQty = 1 } = {}) => {
log.debug('Setting up test environment')
const tracker = createResourceTracker()
@@ -45,7 +46,7 @@ export const setup = async ({ referenceVmId } = {}) => {
await dispatchClient.initialize()
const createdResources = {
vm: null,
vms: [],
backupRepository: null,
sessionId: tracker.getSessionId(),
}
@@ -61,21 +62,23 @@ export const setup = async ({ referenceVmId } = {}) => {
}
const vmPrefix = getRequiredEnv('VM_PREFIX')
const testVmName = await generateIncrementalVmName(dispatchClient, `${vmPrefix}-QA-Test`)
for (let i = 0; i < requiredVmQty; i++) {
const testVmName = await generateIncrementalVmName(dispatchClient, `${vmPrefix}-QA-Test`)
log.debug('Creating test VM', { name: testVmName })
log.debug('Creating test VM', { name: testVmName })
const testVmId = await dispatchClient.vm.clone(referenceVm.uuid, testVmName, {
description: `Test VM for QA tests`,
fastClone: true,
})
const testVmId = await dispatchClient.vm.clone(referenceVm.uuid, testVmName, {
description: `Test VM for QA tests`,
fastClone: true,
})
log.debug('Starting test VM', { id: testVmId })
await dispatchClient.vm.start(testVmId)
await dispatchClient.vm.waitForPowerState(testVmId, 'Running', 120_000)
log.debug('Starting test VM', { id: testVmId })
await dispatchClient.vm.start(testVmId)
await dispatchClient.vm.waitForPowerState(testVmId, 'Running', 120_000)
createdResources.vm = await dispatchClient.vm.details(testVmId)
tracker.trackResource('vm', testVmId, { name: testVmName, source: referenceVm.name_label })
createdResources.vms.push(await dispatchClient.vm.details(testVmId))
tracker.trackResource('vm', testVmId, { name: testVmName, source: referenceVm.name_label })
}
// Create or get backup repository
const backupRepositoryName = getRequiredEnv('BACKUP_REPOSITORY_NAME')

View File

@@ -1,4 +1,5 @@
import { createLogger } from '@xen-orchestra/log'
import { findTaskByMessage } from './index.js'
import assert from 'node:assert'
import { getSyncedHandler } from '@xen-orchestra/fs'
@@ -157,3 +158,52 @@ export const assertBackupSuccess = (result, context = 'Backup') => {
assert.strictEqual(result.status, 'success', `${context} should succeed, got '${result.status}': ${details}`)
}
}
/**
* Asserts that a backup run performed a synchronized snapshot: all VMs were
* snapshotted in a single batch phase, before any transfer, and the per-VM
* backups reused those batched snapshots (no VM was snapshotted twice).
* @param {Object} result - Backup log result
* @param {number} vmCount - Number of VMs concerned by this backup job
*/
export const assertSynchronizedSnapshot = (result, vmCount) => {
const snapshotVmTask = findTaskByMessage(result, 'snapshot VMs')
assert(snapshotVmTask, `Synchronized backup should have batched snapshots`)
assert(
snapshotVmTask.tasks && snapshotVmTask.tasks.length === vmCount,
`Synchronized backup should have made a snapshot per vm`
)
// Collect every task with a given message across the whole task tree.
const collectByMessage = message => {
const found = []
const walk = tasks => {
for (const task of tasks ?? []) {
if (task.message === message) {
found.push(task)
}
if (task.tasks) {
walk(task.tasks)
}
}
}
walk(result.tasks)
return found
}
// Every snapshot must come from the batch phase: there should be exactly one
// 'snapshot' task per VM (the batched ones) and none taken again during the
// per-VM backup. A larger count means a VM was snapshotted a second time at
// transfer time, i.e. the backup did NOT reuse the synchronized snapshot.
const snapshots = collectByMessage('snapshot')
assert(
snapshots.length === vmCount,
`Synchronized backup should reuse the batched snapshots (expected ${vmCount} 'snapshot' tasks, found ${snapshots.length})`
)
const transfers = collectByMessage('transfer')
assert(transfers.length > 0, `Synchronized backup should have transfered snapshots.`)
const earliestStart = Math.min(...transfers.map(t => t.start))
assert(snapshotVmTask.end <= earliestStart, `Synchronized backup snapshots should be complete before transfer starts`)
}

View File

@@ -21,13 +21,12 @@
- [XO6/Host] Add possibility to disable a host an evacuate its VMs (PR [#10090](https://github.com/vatesfr/xen-orchestra/pull/10090))
- [XO6/Host] Add possibility to reboot a host (PR [#10141](https://github.com/vatesfr/xen-orchestra/pull/10141))
- [XO6/Host] Add possibility to force reboot a host (PR [#10175](https://github.com/vatesfr/xen-orchestra/pull/10175))
- [IPMI-plugin] Add GET plugins/ipmi-sensors/hosts/{id}/ipmi to get IPMI sensors (PR [#10003](https://github.com/vatesfr/xen-orchestra/pull/10003))
- [VIF] Add VIF name in header on VIF detail page (PR [#10252](https://github.com/vatesfr/xen-orchestra/pull/10252))
- [IPMI-plugin] Add GET plugins/ipmi-sensors/hosts/{id}/ipmi to get IPMI sensors (PR [#10003](https://github.com/vatesfr/xen-orchestra/pull/10003))
- [VIF] Add VIF name in header on VIF detail page (PR [#10252](https://github.com/vatesfr/xen-orchestra/pull/10252))
- [XO6/SR] Add dedicated Storage Repository page with general information, space usage, PBD details, custom fields (PR [#10100](https://github.com/vatesfr/xen-orchestra/pull/10100))
- [XO5/Backups] Add `Synchronize snapshots` checkbox to backup jobs to get consistent restore points (PR [#10136](https://github.com/vatesfr/xen-orchestra/pull/10136))
### Bug fixes
@@ -61,8 +60,9 @@
- @vates/types patch
- @xen-orchestra/acl minor
- @xen-orchestra/async-map patch
- @xen-orchestra/backups patch
- @xen-orchestra/backups minor
- @xen-orchestra/proxy-cli patch
- @xen-orchestra/qa-test minor
- @xen-orchestra/rest-api minor
- @xen-orchestra/upload-ova patch
- @xen-orchestra/web minor
@@ -74,6 +74,6 @@
- xo-server patch
- xo-server-ipmi-sensors minor
- xo-server-netbox patch
- xo-web patch
- xo-web minor
<!--packages-end-->

View File

@@ -605,6 +605,9 @@ const messages = {
mergeBackupsSynchronously: 'Merge backups synchronously',
mergeBackupsSynchronouslyTooltip:
'This will use more resources on the backup thread, but ensure there is no locking error when chaining multiple backup jobs on the same remote',
synchronizedSnapshot: 'Synchronize snapshots',
synchronizedSnapshotTooltip:
'Snapshots all VMs together before transfer in order to get consistent restore points. Uses more disks, especially on thick-provisioned SRs',
newBackupSelection: 'Select your backup type:',
snapshotRetention: 'Snapshot retention',
backupName: 'Name',

View File

@@ -713,6 +713,11 @@ const New = decorate([
mergeBackupsSynchronously,
})
},
setSynchronizedSnapshot({ setGlobalSettings }, synchronizedSnapshot) {
setGlobalSettings({
synchronizedSnapshot,
})
},
setDistributeBackups({ setGlobalSettings }, distributeBackups) {
setGlobalSettings({
distributeBackups,
@@ -741,6 +746,7 @@ const New = decorate([
inputNbdConcurrency: generateId,
inputNRetriesVmBackupFailures: generateId,
inputPreferNbd: generateId,
inputSynchronizedSnapshot: generateId,
inputDistributeBackups: generateId,
inputDistributeReplications: generateId,
inputTimeoutId: generateId,
@@ -864,6 +870,7 @@ const New = decorate([
preferNbd,
reportRecipients,
reportWhen = 'failure',
synchronizedSnapshot,
distributeBackups = false,
distributeReplications = false,
timeout,
@@ -1318,6 +1325,21 @@ const New = decorate([
onChange={effects.setMergeBackupsSynchronously}
/>
</FormGroup>
<FormGroup>
<label htmlFor={state.inputSynchronizedSnapshot}>
<strong>{_('synchronizedSnapshot')}</strong>
</label>{' '}
<Tooltip content={_('synchronizedSnapshotTooltip')}>
<Icon icon='info' />
</Tooltip>
<Toggle
className='pull-right'
id={state.inputSynchronizedSnapshot}
name='synchronizedSnapshot'
value={synchronizedSnapshot === true}
onChange={effects.setSynchronizedSnapshot}
/>
</FormGroup>
</div>
)}
</CardBlock>