feat(backup-ng): offline snapshotless backup (#4470)

Fixes #3449
This commit is contained in:
badrAZ
2019-10-25 09:45:45 +02:00
committed by Julien Fontanet
parent 0c453c4415
commit a1a53bb285
10 changed files with 431 additions and 149 deletions

View File

@@ -17,6 +17,7 @@
- [xo-server-transport-icinga2] Add support of [icinga2](https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/) for reporting services status [#4563](https://github.com/vatesfr/xen-orchestra/issues/4563) (PR [#4573](https://github.com/vatesfr/xen-orchestra/pull/4573))
- [Hub] Ability to update existing template (PR [#4613](https://github.com/vatesfr/xen-orchestra/pull/4613))
- [Menu] Remove legacy backup entry [#4467](https://github.com/vatesfr/xen-orchestra/issues/4467) (PR [#4476](https://github.com/vatesfr/xen-orchestra/pull/4476))
- [Backup NG] Offline backup feature [#3449](https://github.com/vatesfr/xen-orchestra/issues/3449) (PR [#4470](https://github.com/vatesfr/xen-orchestra/pull/4470))
### Bug fixes

View File

@@ -14,6 +14,7 @@
[vms]
default = ''
withOsAndXenTools = ''
# vmToBackup = ''
[templates]

View File

@@ -154,6 +154,19 @@ class XoConnection extends Xo {
})
}
async startTempVm(id, params, withXenTools = false) {
await this.call('vm.start', { id, ...params })
this._tempResourceDisposers.push('vm.stop', { id, force: true })
return this.waitObjectState(id, vm => {
if (
vm.power_state !== 'Running' ||
(withXenTools && vm.xenTools === false)
) {
throw new Error('retry')
}
})
}
async createTempRemote(params) {
const remote = await this.call('remote.create', params)
this._tempResourceDisposers.push('remote.delete', { id: remote.id })

View File

@@ -55,6 +55,68 @@ Object {
}
`;
exports[`backupNg create and execute backup with enabled offline backup 1`] = `
Object {
"data": Object {
"id": Any<String>,
"type": "VM",
},
"end": Any<Number>,
"id": Any<String>,
"message": Any<String>,
"start": Any<Number>,
"status": "success",
}
`;
exports[`backupNg create and execute backup with enabled offline backup 2`] = `
Object {
"data": Any<Object>,
"end": Any<Number>,
"id": Any<String>,
"message": Any<String>,
"start": Any<Number>,
"status": "success",
}
`;
exports[`backupNg create and execute backup with enabled offline backup 3`] = `
Object {
"end": Any<Number>,
"id": Any<String>,
"message": Any<String>,
"result": Object {
"size": Any<Number>,
},
"start": Any<Number>,
"status": "success",
}
`;
exports[`backupNg create and execute backup with enabled offline backup 4`] = `
Object {
"data": Any<Object>,
"end": Any<Number>,
"id": Any<String>,
"message": Any<String>,
"start": Any<Number>,
"status": "success",
}
`;
exports[`backupNg create and execute backup with enabled offline backup 5`] = `
Object {
"end": Any<Number>,
"id": Any<String>,
"message": Any<String>,
"result": Object {
"size": Any<Number>,
},
"start": Any<Number>,
"status": "success",
}
`;
exports[`backupNg execute three times a delta backup with 2 remotes, 2 as retention and 2 as fullInterval 1`] = `
Object {
"data": Object {

View File

@@ -584,4 +584,110 @@ describe('backupNg', () => {
})
})
})
test('create and execute backup with enabled offline backup', async () => {
const vm = xo.objects.all[config.vms.withOsAndXenTools]
if (vm.power_state !== 'Running') {
await xo.startTempVm(vm.id, { force: true }, true)
}
const scheduleTempId = randomId()
const srId = config.srs.default
const { id: remoteId } = await xo.createTempRemote(config.remotes.default)
const backupInput = {
mode: 'full',
remotes: {
id: remoteId,
},
schedules: {
[scheduleTempId]: getDefaultSchedule(),
},
settings: {
'': {
offlineBackup: true,
},
[scheduleTempId]: {
copyRetention: 1,
exportRetention: 1,
},
},
srs: {
id: srId,
},
vms: {
id: vm.id,
},
}
const backup = await xo.createTempBackupNgJob(backupInput)
expect(backup.settings[''].offlineBackup).toBe(true)
const schedule = await xo.getSchedule({ jobId: backup.id })
await Promise.all([
xo.runBackupJob(backup.id, schedule.id, { remotes: [remoteId] }),
xo.waitObjectState(vm.id, vm => {
if (vm.power_state !== 'Halted') {
throw new Error('retry')
}
}),
])
await xo.waitObjectState(vm.id, vm => {
if (vm.power_state !== 'Running') {
throw new Error('retry')
}
})
const backupLogs = await xo.getBackupLogs({
jobId: backup.id,
scheduleId: schedule.id,
})
expect(backupLogs.length).toBe(1)
const { tasks, ...log } = backupLogs[0]
validateRootTask(log, {
data: {
mode: backupInput.mode,
reportWhen: backupInput.settings[''].reportWhen,
},
jobId: backup.id,
jobName: backupInput.name,
scheduleId: schedule.id,
status: 'success',
})
expect(Array.isArray(tasks)).toBe(true)
tasks.forEach(({ tasks, ...vmTask }) => {
validateVmTask(vmTask, vm.id, { status: 'success' })
expect(Array.isArray(tasks)).toBe(true)
tasks.forEach(({ tasks, ...subTask }) => {
expect(subTask.message).not.toBe('snapshot')
if (subTask.message === 'export') {
validateExportTask(
subTask,
subTask.data.type === 'remote' ? remoteId : srId,
{
data: expect.any(Object),
status: 'success',
}
)
expect(Array.isArray(tasks)).toBe(true)
tasks.forEach(operationTask => {
if (
operationTask.message === 'transfer' ||
operationTask.message === 'merge'
) {
validateOperationTask(operationTask, {
result: { size: expect.any(Number) },
status: 'success',
})
}
})
}
})
})
}, 200e3)
})

View File

@@ -53,7 +53,7 @@ import {
type Xapi,
TAG_COPY_SRC,
} from '../../xapi'
import { getVmDisks } from '../../xapi/utils'
import { formatDateTime, getVmDisks } from '../../xapi/utils'
import {
resolveRelativeFromFile,
safeDateFormat,
@@ -75,6 +75,7 @@ type Settings = {|
deleteFirst?: boolean,
copyRetention?: number,
exportRetention?: number,
offlineBackup?: boolean,
offlineSnapshot?: boolean,
reportWhen?: ReportWhen,
snapshotRetention?: number,
@@ -147,6 +148,7 @@ const defaultSettings: Settings = {
deleteFirst: false,
exportRetention: 0,
fullInterval: 0,
offlineBackup: false,
offlineSnapshot: false,
reportWhen: 'failure',
snapshotRetention: 0,
@@ -479,16 +481,21 @@ const disableVmHighAvailability = async (xapi: Xapi, vm: Vm) => {
// Attributes on created VM snapshots:
//
// - `other_config`:
// - `xo:backup:datetime` = snapshot.snapshot_time (allow sorting replicated VMs)
// - `xo:backup:deltaChainLength` = n (number of delta copies/replicated since a full)
// - `xo:backup:exported` = 'true' (added at the end of the backup)
//
// Attributes on created VMs and created snapshots:
//
// - `other_config`:
// - `xo:backup:datetime`: format is UTC %Y%m%dT%H:%M:%SZ
// - from snapshots: snapshot.snapshot_time
// - with offline backup: formatDateTime(Date.now())
// - `xo:backup:job` = job.id
// - `xo:backup:schedule` = schedule.id
// - `xo:backup:vm` = vm.uuid
//
// Attributes of created VMs:
//
// - all snapshots attributes (see above)
// - `name_label`: `${original name} - ${job name} - (${safeDateFormat(backup timestamp)})`
// - tag:
// - copy in delta mode: `Continuous Replication`
@@ -1023,6 +1030,12 @@ export default class BackupNg {
throw new Error('copy, export and snapshot retentions cannot both be 0')
}
const isOfflineBackup =
mode === 'full' && getSetting(settings, 'offlineBackup', [vmUuid, ''])
if (isOfflineBackup && snapshotRetention > 0) {
throw new Error('offline backup is not compatible with rolling snapshot')
}
if (
!some(
vm.$VBDs,
@@ -1032,110 +1045,139 @@ export default class BackupNg {
throw new Error('no disks found')
}
const snapshots = vm.$snapshots
.filter(_ => _.other_config['xo:backup:job'] === jobId)
.sort(compareSnapshotTime)
let baseSnapshot, exported: Vm, exportDateTime
if (isOfflineBackup) {
exported = vm
exportDateTime = formatDateTime(Date.now())
if (vm.power_state === 'Running') {
await wrapTask(
{
logger,
message: 'shutdown VM',
parentId: taskId,
},
xapi.shutdownVm(vm)
)
$defer(() => xapi.startVm(vm))
}
} else {
const snapshots = vm.$snapshots
.filter(_ => _.other_config['xo:backup:job'] === jobId)
.sort(compareSnapshotTime)
const bypassVdiChainsCheck: boolean = getSetting(
settings,
'bypassVdiChainsCheck',
[vmUuid, '']
)
if (!bypassVdiChainsCheck) {
xapi._assertHealthyVdiChains(vm)
}
const bypassVdiChainsCheck: boolean = getSetting(
settings,
'bypassVdiChainsCheck',
[vmUuid, '']
)
if (!bypassVdiChainsCheck) {
xapi._assertHealthyVdiChains(vm)
}
const offlineSnapshot: boolean = getSetting(settings, 'offlineSnapshot', [
vmUuid,
'',
])
const startAfterSnapshot = offlineSnapshot && vm.power_state === 'Running'
if (startAfterSnapshot) {
await wrapTask(
{
logger,
message: 'shutdown VM',
parentId: taskId,
},
xapi.shutdownVm(vm)
)
}
exported = (await wrapTask(
{
logger,
message: 'snapshot',
parentId: taskId,
result: _ => _.uuid,
},
xapi._snapshotVm(
$cancelToken,
vm,
`[XO Backup ${job.name}] ${vm.name_label}`
)
): any)
if (startAfterSnapshot) {
ignoreErrors.call(xapi.startVm(vm))
}
const offlineSnapshot: boolean = getSetting(settings, 'offlineSnapshot', [
vmUuid,
'',
])
const startAfterSnapshot = offlineSnapshot && vm.power_state === 'Running'
if (startAfterSnapshot) {
await wrapTask(
{
logger,
message: 'shutdown VM',
message: 'add metadata to snapshot',
parentId: taskId,
},
xapi.shutdownVm(vm)
)
}
let snapshot: Vm = (await wrapTask(
{
logger,
message: 'snapshot',
parentId: taskId,
result: _ => _.uuid,
},
xapi._snapshotVm(
$cancelToken,
vm,
`[XO Backup ${job.name}] ${vm.name_label}`
)
): any)
if (startAfterSnapshot) {
ignoreErrors.call(xapi.startVm(vm))
}
await wrapTask(
{
logger,
message: 'add metadata to snapshot',
parentId: taskId,
},
snapshot.update_other_config({
'xo:backup:datetime': snapshot.snapshot_time,
'xo:backup:job': jobId,
'xo:backup:schedule': scheduleId,
'xo:backup:vm': vmUuid,
})
)
snapshot = await xapi.barrier(snapshot.$ref)
let baseSnapshot
if (mode === 'delta') {
baseSnapshot = findLast(
snapshots,
_ => 'xo:backup:exported' in _.other_config
exported.update_other_config({
'xo:backup:datetime': exported.snapshot_time,
'xo:backup:job': jobId,
'xo:backup:schedule': scheduleId,
'xo:backup:vm': vmUuid,
})
)
// JFT 2018-10-02: support previous snapshots which did not have this
// entry, can be removed after 2018-12.
if (baseSnapshot === undefined) {
baseSnapshot = last(snapshots)
}
}
snapshots.push(snapshot)
exported = await xapi.barrier(exported.$ref)
// snapshots to delete due to the snapshot retention settings
const snapshotsToDelete = flatMap(
groupBy(snapshots, _ => _.other_config['xo:backup:schedule']),
(snapshots, scheduleId) =>
getOldEntries(
getSetting(settings, 'snapshotRetention', [scheduleId]),
snapshots
if (mode === 'delta') {
baseSnapshot = findLast(
snapshots,
_ => 'xo:backup:exported' in _.other_config
)
)
// delete unused snapshots
await asyncMap(snapshotsToDelete, vm => {
// snapshot and baseSnapshot should not be deleted right now
if (vm !== snapshot && vm !== baseSnapshot) {
return xapi.deleteVm(vm)
// JFT 2018-10-02: support previous snapshots which did not have this
// entry, can be removed after 2018-12.
if (baseSnapshot === undefined) {
baseSnapshot = last(snapshots)
}
}
})
snapshots.push(exported)
snapshot = ((await wrapTask(
{
logger,
message: 'waiting for uptodate snapshot record',
parentId: taskId,
},
xapi.barrier(snapshot.$ref)
): any): Vm)
// snapshots to delete due to the snapshot retention settings
const snapshotsToDelete = flatMap(
groupBy(snapshots, _ => _.other_config['xo:backup:schedule']),
(snapshots, scheduleId) =>
getOldEntries(
getSetting(settings, 'snapshotRetention', [scheduleId]),
snapshots
)
)
// delete unused snapshots
await asyncMap(snapshotsToDelete, vm => {
// snapshot and baseSnapshot should not be deleted right now
if (vm !== exported && vm !== baseSnapshot) {
return xapi.deleteVm(vm)
}
})
exported = ((await wrapTask(
{
logger,
message: 'waiting for uptodate snapshot record',
parentId: taskId,
},
xapi.barrier(exported.$ref)
): any): Vm)
if (mode === 'full' && snapshotsToDelete.includes(exported)) {
// TODO: do not create the snapshot if there are no snapshotRetention and
// the VM is not running
$defer.call(xapi, 'deleteVm', exported)
} else if (mode === 'delta') {
if (snapshotsToDelete.includes(exported)) {
$defer.onFailure.call(xapi, 'deleteVm', exported)
}
if (snapshotsToDelete.includes(baseSnapshot)) {
$defer.onSuccess.call(xapi, 'deleteVm', baseSnapshot)
}
}
}
if (copyRetention === 0 && exportRetention === 0) {
return
@@ -1151,14 +1193,8 @@ export default class BackupNg {
const metadataFilename = `${vmDir}/${basename}.json`
if (mode === 'full') {
// TODO: do not create the snapshot if there are no snapshotRetention and
// the VM is not running
if (snapshotsToDelete.includes(snapshot)) {
$defer.call(xapi, 'deleteVm', snapshot)
}
let compress = getJobCompression(job)
const pool = snapshot.$pool
const pool = exported.$pool
if (
compress === 'zstd' &&
pool.restrictions.restrict_zstd_export !== 'false'
@@ -1175,10 +1211,10 @@ export default class BackupNg {
let xva: any = await wrapTask(
{
logger,
message: 'start snapshot export',
message: 'start VM export',
parentId: taskId,
},
xapi.exportVm($cancelToken, snapshot, {
xapi.exportVm($cancelToken, exported, {
compress,
})
)
@@ -1203,7 +1239,7 @@ export default class BackupNg {
timestamp: now,
version: '2.0.0',
vm,
vmSnapshot: snapshot,
vmSnapshot: exported.id !== vm.id ? exported : undefined,
xva: `./${dataBasename}`,
}
const dataFilename = `${vmDir}/${dataBasename}`
@@ -1343,7 +1379,15 @@ export default class BackupNg {
'start',
'Start operation for this vm is blocked, clone it if you want to use it.'
),
vm.update_other_config('xo:backup:sr', srUuid),
!isOfflineBackup
? vm.update_other_config('xo:backup:sr', srUuid)
: vm.update_other_config({
'xo:backup:datetime': exportDateTime,
'xo:backup:job': jobId,
'xo:backup:schedule': scheduleId,
'xo:backup:sr': srUuid,
'xo:backup:vm': exported.uuid,
}),
])
if (!deleteFirst) {
@@ -1356,13 +1400,6 @@ export default class BackupNg {
noop // errors are handled in logs
)
} else if (mode === 'delta') {
if (snapshotsToDelete.includes(snapshot)) {
$defer.onFailure.call(xapi, 'deleteVm', snapshot)
}
if (snapshotsToDelete.includes(baseSnapshot)) {
$defer.onSuccess.call(xapi, 'deleteVm', baseSnapshot)
}
let deltaChainLength = 0
let fullVdisRequired
await (async () => {
@@ -1470,7 +1507,7 @@ export default class BackupNg {
message: 'start snapshot export',
parentId: taskId,
},
xapi.exportDeltaVm($cancelToken, snapshot, baseSnapshot, {
xapi.exportDeltaVm($cancelToken, exported, baseSnapshot, {
fullVdisRequired,
})
)
@@ -1492,7 +1529,7 @@ export default class BackupNg {
}/${basename}.vhd`
),
vm,
vmSnapshot: snapshot,
vmSnapshot: exported,
}
const jsonMetadata = JSON.stringify(metadata)
@@ -1728,7 +1765,7 @@ export default class BackupNg {
if (!isFull) {
ignoreErrors.call(
snapshot.update_other_config(
exported.update_other_config(
'xo:backup:deltaChainLength',
String(deltaChainLength)
)
@@ -1738,14 +1775,16 @@ export default class BackupNg {
throw new Error(`no exporter for backup mode ${mode}`)
}
await wrapTask(
{
logger,
message: 'set snapshot.other_config[xo:backup:exported]',
parentId: taskId,
},
snapshot.update_other_config('xo:backup:exported', 'true')
)
if (!isOfflineBackup) {
await wrapTask(
{
logger,
message: 'set snapshot.other_config[xo:backup:exported]',
parentId: taskId,
},
exported.update_other_config('xo:backup:exported', 'true')
)
}
}
async _deleteDeltaVmBackups(

View File

@@ -453,6 +453,9 @@ const messages = {
backupName: 'Name',
offlineSnapshot: 'Offline snapshot',
offlineSnapshotInfo: 'Shutdown VMs before snapshotting them',
offlineBackup: 'Offline backup',
offlineBackupInfo:
'Export VMs without snapshotting them. The VMs will be shutdown during the export.',
timeout: 'Timeout',
timeoutInfo: 'Number of hours after which a job is considered failed',
fullBackupInterval: 'Full backup interval',

View File

@@ -6,6 +6,7 @@ const DEFAULTS = {
compression: '',
concurrency: 0,
fullInterval: 0,
offlineBackup: false,
offlineSnapshot: false,
reportWhen: 'failure',
timeout: 0,
@@ -16,6 +17,7 @@ const MODES = {
compression: 'full',
fullInterval: 'delta',
offlineBackup: 'full',
}
const getSettingsWithNonDefaultValue = (mode, settings) =>

View File

@@ -90,7 +90,13 @@ const ThinProvisionedTip = ({ label }) => (
const normalizeTagValues = values => resolveIds(values).map(value => [value])
const normalizeSettings = ({ settings, exportMode, copyMode, snapshotMode }) =>
const normalizeSettings = ({
copyMode,
exportMode,
offlineBackupActive,
settings,
snapshotMode,
}) =>
settings.map(setting =>
defined(
setting.copyRetention,
@@ -100,9 +106,10 @@ const normalizeSettings = ({ settings, exportMode, copyMode, snapshotMode }) =>
? {
copyRetention: copyMode ? setting.copyRetention : undefined,
exportRetention: exportMode ? setting.exportRetention : undefined,
snapshotRetention: snapshotMode
? setting.snapshotRetention
: undefined,
snapshotRetention:
snapshotMode && !offlineBackupActive
? setting.snapshotRetention
: undefined,
}
: setting
)
@@ -200,6 +207,7 @@ export default decorate([
({ id, ...schedule }) => schedule
)
settings = normalizeSettings({
offlineBackupActive: state.offlineBackupActive,
settings: state.settings,
exportMode: state.exportMode,
copyMode: state.copyMode,
@@ -211,12 +219,14 @@ export default decorate([
[id]: DEFAULT_SCHEDULE,
}
settings = {
'': state.settings && state.settings.get(''),
[id]: {
copyRetention: state.copyMode ? DEFAULT_RETENTION : undefined,
exportRetention: state.exportMode ? DEFAULT_RETENTION : undefined,
snapshotRetention: state.snapshotMode
? DEFAULT_RETENTION
: undefined,
snapshotRetention:
state.snapshotMode && !state.offlineBackupActive
? DEFAULT_RETENTION
: undefined,
},
}
}
@@ -300,6 +310,7 @@ export default decorate([
mode: state.isDelta ? 'delta' : 'full',
compression: state.compression,
settings: normalizeSettings({
offlineBackupActive: state.offlineBackupActive,
settings: settings || state.propSettings,
exportMode: state.exportMode,
copyMode: state.copyMode,
@@ -545,6 +556,15 @@ export default decorate([
value,
})
},
setOfflineBackup: (
{ setGlobalSettings },
{ target: { checked: value } }
) => () => {
setGlobalSettings({
name: 'offlineBackup',
value,
})
},
},
computed: {
compressionId: generateId,
@@ -553,6 +573,13 @@ export default decorate([
inputFullIntervalId: generateId,
inputTimeoutId: generateId,
// In order to keep the user preference, the offline backup is kept in the DB
// and it's considered active only when the full mode is enabled
offlineBackupActive: ({
isFull,
propSettings,
settings = propSettings,
}) => isFull && Boolean(settings.getIn(['', 'offlineBackup'])),
vmsPattern: ({ _vmsPattern }, { job }) =>
defined(
_vmsPattern,
@@ -655,6 +682,7 @@ export default decorate([
const {
concurrency,
fullInterval,
offlineBackup,
offlineSnapshot,
reportWhen = 'failure',
timeout,
@@ -691,6 +719,7 @@ export default decorate([
<ActionButton
active={state.snapshotMode}
data-mode='snapshotMode'
disabled={state.offlineBackupActive}
handler={effects.toggleMode}
icon='rolling-snapshot'
>
@@ -943,30 +972,47 @@ export default decorate([
</FormGroup>
)}
{state.isFull && (
<div>
<FormGroup>
<label htmlFor={state.compressionId}>
<strong>{_('compression')}</strong>
</label>
<SelectCompression
id={state.compressionId}
onChange={effects.setCompression}
value={compression}
/>
</FormGroup>
<FormGroup>
<label>
<strong>{_('offlineBackup')}</strong>{' '}
<Tooltip content={_('offlineBackupInfo')}>
<Icon icon='info' />
</Tooltip>{' '}
<input
checked={offlineBackup}
onChange={effects.setOfflineBackup}
type='checkbox'
/>
</label>
</FormGroup>
</div>
)}
{!state.offlineBackupActive && (
<FormGroup>
<label htmlFor={state.compressionId}>
<strong>{_('compression')}</strong>
<label>
<strong>{_('offlineSnapshot')}</strong>{' '}
<Tooltip content={_('offlineSnapshotInfo')}>
<Icon icon='info' />
</Tooltip>{' '}
<input
checked={offlineSnapshot}
onChange={effects.setOfflineSnapshot}
type='checkbox'
/>
</label>
<SelectCompression
id={state.compressionId}
onChange={effects.setCompression}
value={compression}
/>
</FormGroup>
)}
<FormGroup>
<label>
<strong>{_('offlineSnapshot')}</strong>{' '}
<Tooltip content={_('offlineSnapshotInfo')}>
<Icon icon='info' />
</Tooltip>{' '}
<input
checked={offlineSnapshot}
onChange={effects.setOfflineSnapshot}
type='checkbox'
/>
</label>
</FormGroup>
</div>
)}
</CardBlock>

View File

@@ -242,6 +242,7 @@ class JobsTable extends React.Component {
compression,
concurrency,
fullInterval,
offlineBackup,
offlineSnapshot,
reportWhen,
timeout,
@@ -264,6 +265,14 @@ class JobsTable extends React.Component {
{fullInterval !== undefined && (
<Li>{_.keyValue(_('fullBackupInterval'), fullInterval)}</Li>
)}
{offlineBackup !== undefined && (
<Li>
{_.keyValue(
_('offlineBackup'),
_(offlineBackup ? 'stateEnabled' : 'stateDisabled')
)}
</Li>
)}
{offlineSnapshot !== undefined && (
<Li>
{_.keyValue(