mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-09 17:06:13 -06:00
test(backups): replication and full mirror backups
* test(replication): test the new snapshot based replication * fix(tests): use mergeBackupsSynchronously to ensure the backups don't fails with lock already heled error * fix: ensure the repository that will be purged start empty * test(backup): add test for full mirror backups
This commit is contained in:
Florent BEAUCHAMP
committed by
GitHub
GitHub
parent
8df8ef0af8
commit
5f7fab9d34
@@ -22,6 +22,15 @@ BACKUP_REPOSITORY_PATH=/tmp/xo-test-backups
|
||||
# This path is NOT included in automatic cleanup to protect production data
|
||||
BACKUP_PATH=/var/lib/xo/backups
|
||||
|
||||
# ===========================================
|
||||
# MIRROR BACKUP TEST CONFIGURATION
|
||||
# ===========================================
|
||||
|
||||
# Destination repository for mirror backup tests
|
||||
# Source is BACKUP_REPOSITORY_NAME above
|
||||
MIRROR_DESTINATION_REPOSITORY_NAME=Test mirror QA
|
||||
MIRROR_DESTINATION_REPOSITORY_PATH=/tmp/xo-test-mirror
|
||||
|
||||
# ===========================================
|
||||
# VHD/XVA EXPORT TEST CONFIGURATION
|
||||
# ===========================================
|
||||
|
||||
@@ -22,6 +22,7 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {})
|
||||
'': {
|
||||
timezone: 'Europe/Paris',
|
||||
exportRetention: 2,
|
||||
mergeBackupsSynchronously: true,
|
||||
},
|
||||
},
|
||||
vms: {
|
||||
|
||||
@@ -272,7 +272,12 @@ export class CleanupClient {
|
||||
}
|
||||
}
|
||||
|
||||
await this.dispatchClient.backup.deleteBackupJob(job.id)
|
||||
// Mirror backup jobs require mirrorBackup.deleteJob, standard jobs use backupNg.deleteJob
|
||||
if (job.type === 'mirrorBackup') {
|
||||
await this.dispatchClient.backup.deleteMirrorBackupJob(job.id)
|
||||
} else {
|
||||
await this.dispatchClient.backup.deleteBackupJob(job.id)
|
||||
}
|
||||
}
|
||||
|
||||
return await this._deleteResourcesBatch('backup job', findJobs, deleteJob, {
|
||||
|
||||
@@ -38,22 +38,31 @@ export class BackupRequest extends AbstractRequest {
|
||||
|
||||
// Convert dynamic key format to XO expected format
|
||||
const convertConfig = cfg => {
|
||||
const { vms, remotes, ...rest } = cfg
|
||||
const { vms, remotes, srs, ...rest } = cfg
|
||||
|
||||
// Extract first VM UUID
|
||||
const vmUuid = vms && typeof vms === 'object' ? Object.keys(vms)[0] : undefined
|
||||
// Extract first Backup Repository ID
|
||||
const backupRepositoryId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined
|
||||
|
||||
return {
|
||||
const result = {
|
||||
...rest,
|
||||
vms: {
|
||||
id: vmUuid,
|
||||
},
|
||||
remotes: {
|
||||
id: backupRepositoryId,
|
||||
},
|
||||
}
|
||||
|
||||
// Extract first Backup Repository ID (for backup to remote)
|
||||
const backupRepositoryId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined
|
||||
if (backupRepositoryId !== undefined) {
|
||||
result.remotes = { id: backupRepositoryId }
|
||||
}
|
||||
|
||||
// Extract first SR ID (for CR/DR mode — replication to SR)
|
||||
const srId = srs && typeof srs === 'object' ? Object.keys(srs)[0] : undefined
|
||||
if (srId !== undefined) {
|
||||
result.srs = { id: srId }
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
const xoConfig = convertConfig({ ...config, mode: config.mode || 'delta' })
|
||||
@@ -155,6 +164,139 @@ export class BackupRequest extends AbstractRequest {
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Mirror Backup operations (mirrorBackup.* API)
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Creates a new mirror backup job in XenOrchestra.
|
||||
*
|
||||
* Mirror backup replicates backups from one remote to another.
|
||||
* Uses `mirrorBackup.createJob` WebSocket API.
|
||||
*
|
||||
* @param {Object} config - Mirror backup job configuration
|
||||
* @param {string} config.mode - 'full' or 'delta'
|
||||
* @param {string} config.sourceRemote - ID of the source remote
|
||||
* @param {Object} config.remotes - Destination remotes (e.g., { [remoteId]: {} })
|
||||
* @param {string} [config.name] - Job name
|
||||
* @param {Object} [config.schedules] - Schedule definitions
|
||||
* @param {Object} [config.settings] - Job settings
|
||||
* @param {Object} [config.filter] - VM filter (by UUID)
|
||||
* @returns {Promise<string>} The created mirror backup job ID
|
||||
*/
|
||||
async createMirrorBackupJob(config) {
|
||||
this._ensureConnected()
|
||||
|
||||
if (!config || typeof config !== 'object') {
|
||||
throw new Error('Valid mirror backup configuration object is required')
|
||||
}
|
||||
|
||||
const { remotes, ...rest } = config
|
||||
|
||||
// Convert remotes to { id: remoteId } format
|
||||
const remoteId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined
|
||||
|
||||
const xoConfig = {
|
||||
...rest,
|
||||
mode: config.mode || 'full',
|
||||
}
|
||||
|
||||
if (remoteId !== undefined) {
|
||||
xoConfig.remotes = { id: remoteId }
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.dispatchClient.xoClient.call('mirrorBackup.createJob', xoConfig)
|
||||
return result
|
||||
} catch (error) {
|
||||
console.error('❌ Mirror backup job creation failed:', {
|
||||
message: error.message,
|
||||
code: error.code,
|
||||
data: error.data,
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a mirror backup job.
|
||||
*
|
||||
* @param {string} jobId - ID of the mirror backup job to delete
|
||||
* @returns {Promise<Object>} Deletion result
|
||||
*/
|
||||
async deleteMirrorBackupJob(jobId) {
|
||||
this._ensureConnected()
|
||||
|
||||
if (!jobId || typeof jobId !== 'string') {
|
||||
throw new Error('Valid mirror backup job ID is required')
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.dispatchClient.xoClient.call('mirrorBackup.deleteJob', { id: jobId })
|
||||
return result
|
||||
} catch (error) {
|
||||
console.error('❌ Delete mirror backup job failed:', error.message)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a mirror backup job and monitors completion.
|
||||
*
|
||||
* @param {string} jobId - Mirror backup job ID to execute
|
||||
* @param {string} scheduleId - Schedule ID to use for this execution
|
||||
* @returns {Promise<Object>} Complete backup log with status and details
|
||||
*/
|
||||
async runMirrorJobAndGetLog(jobId, scheduleId) {
|
||||
this._ensureConnected()
|
||||
|
||||
console.log(`🚀 Running mirror backup job ${jobId} with schedule ${scheduleId}...`)
|
||||
|
||||
const runStartTime = Date.now() - 5000
|
||||
|
||||
try {
|
||||
await this.dispatchClient.xoClient.call('mirrorBackup.runJob', {
|
||||
id: jobId,
|
||||
schedule: scheduleId,
|
||||
})
|
||||
} catch (wsError) {
|
||||
console.error('❌ Run mirror backup job failed:', wsError.message)
|
||||
throw wsError
|
||||
}
|
||||
|
||||
const backupLog = await waitUntil(
|
||||
async () => {
|
||||
try {
|
||||
const latestLog = await this.dispatchClient.backupLog.getLatestForJob(jobId, runStartTime)
|
||||
|
||||
if (!latestLog) {
|
||||
return false
|
||||
}
|
||||
|
||||
const status = latestLog.status?.toLowerCase()
|
||||
if (status === 'success' || status === 'failure' || latestLog.end) {
|
||||
return latestLog
|
||||
}
|
||||
|
||||
return false
|
||||
} catch (error) {
|
||||
console.debug('Waiting for mirror backup completion...', error.message)
|
||||
return false
|
||||
}
|
||||
},
|
||||
2000,
|
||||
300_000,
|
||||
{ exponentialBackoff: true, backoffMultiplier: 1.2, maxInterval: 10_000 }
|
||||
)
|
||||
|
||||
console.log(`✅ Mirror backup completed: ${backupLog.status}`)
|
||||
return backupLog
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Standard Backup operations (backupNg.* API)
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Executes a backup job and monitors completion.
|
||||
*
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
"qa:backup:nbd": "node --env-file-if-exists=.env --test tests/backup.nbd.test.js",
|
||||
"qa:backup:combined": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js",
|
||||
"qa:backup:combined:cycle": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='Full Cycle'",
|
||||
"qa:backup:combined:cr": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='CR Mode'",
|
||||
"qa:mirror": "node --env-file-if-exists=.env --test tests/backup-mirror.test.js",
|
||||
"qa:export": "node --env-file-if-exists=.env --test tests/export.vhd.test.js",
|
||||
"qa:export:vhd": "node --env-file-if-exists=.env --test tests/export.vhd.test.js --test-name-pattern='VDI VHD'",
|
||||
"qa:export:xva": "node --env-file-if-exists=.env --test tests/export.vhd.test.js --test-name-pattern='XVA'",
|
||||
|
||||
344
@xen-orchestra/qa-test/tests/backup-mirror.test.js
Normal file
344
@xen-orchestra/qa-test/tests/backup-mirror.test.js
Normal file
@@ -0,0 +1,344 @@
|
||||
import assert from 'node:assert'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
|
||||
import { backupConfig } from '../backup.config.js'
|
||||
import {
|
||||
assertFullOrDelta,
|
||||
findTaskByMessage,
|
||||
generateBackupJobName,
|
||||
getBackupTransferredBytes,
|
||||
getDefaultSchedule,
|
||||
getScheduleKey,
|
||||
} from '../utils/index.js'
|
||||
import { assertBackupSuccess, assertRepositoryEmpty } from '../utils/backupUtils.js'
|
||||
import { setup, teardown } from './setup.js'
|
||||
|
||||
/** Source backup repository — where initial backups are stored */
|
||||
const SOURCE_REPOSITORY_NAME = process.env.BACKUP_REPOSITORY_NAME || 'Test backup QA'
|
||||
|
||||
/** Destination backup repository — mirror target */
|
||||
const MIRROR_DESTINATION_REPOSITORY_NAME = process.env.MIRROR_DESTINATION_REPOSITORY_NAME || 'Test mirror QA'
|
||||
const MIRROR_DESTINATION_REPOSITORY_PATH = process.env.MIRROR_DESTINATION_REPOSITORY_PATH
|
||||
|
||||
describe('Mirror Backup - Full Remote', () => {
|
||||
/** @type {import('../client/dispatchClient.js').DispatchClient} */
|
||||
let dispatchClient
|
||||
/** @type {import('../utils/resourceTracker.js').ResourceTracker} */
|
||||
let tracker
|
||||
/** @type {{uuid: string, name_label: string, power_state: string}} */
|
||||
let vm
|
||||
/** @type {{id: string, name: string}} */
|
||||
let sourceRepository
|
||||
/** @type {{id: string, name: string}} */
|
||||
let destRepository
|
||||
|
||||
/**
|
||||
* Counts backups for a specific VM on a given repository.
|
||||
* @param {string} repoId - Repository ID
|
||||
* @param {string} vmUuid - VM UUID to count backups for
|
||||
* @returns {Promise<number>} Number of backups
|
||||
*/
|
||||
const countVmBackups = async (repoId, vmUuid) => {
|
||||
const backups = await dispatchClient.backup.listVmBackups([repoId])
|
||||
const repoBackups = backups[repoId] ?? backups[Object.keys(backups)[0]] ?? {}
|
||||
const vmBackups = repoBackups[vmUuid] ?? []
|
||||
return vmBackups.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Purges all VM backup data from one or more repositories.
|
||||
* @param {...{id: string, name: string}} repos - Repositories to purge
|
||||
*/
|
||||
const purgeBackupData = async (...repos) => {
|
||||
for (const repo of repos) {
|
||||
if (!repo) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const backups = await dispatchClient.backup.listVmBackups([repo.id])
|
||||
const ids = []
|
||||
for (const vmBackups of Object.values(backups)) {
|
||||
for (const backupList of Object.values(vmBackups)) {
|
||||
for (const backup of backupList) {
|
||||
ids.push(backup.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ids.length > 0) {
|
||||
console.log(`Purging ${ids.length} backup(s) from ${repo.name}...`)
|
||||
await dispatchClient.backup.deleteVmBackups(ids)
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Backup purge failed for ${repo.name}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a backup job, runs it, and returns the result.
|
||||
* Tracks the job and schedule for cleanup.
|
||||
*/
|
||||
const runFullBackup = async () => {
|
||||
const name = generateBackupJobName()
|
||||
const schedule = getDefaultSchedule()
|
||||
const cfg = backupConfig(name, schedule, vm, sourceRepository)
|
||||
|
||||
const jobId = await dispatchClient.backup.createBackupJob({ ...cfg, mode: 'full' })
|
||||
const job = await dispatchClient.backup.details(jobId)
|
||||
tracker.trackResource('backupJob', jobId, { name, mode: 'full' })
|
||||
|
||||
const scheduleKey = getScheduleKey(job)
|
||||
assert.ok(scheduleKey, 'Backup schedule key is required')
|
||||
tracker.trackResource('schedule', scheduleKey, { name, backupJobId: jobId })
|
||||
|
||||
const result = await dispatchClient.backup.runJobAndGetLog(jobId, scheduleKey)
|
||||
assertBackupSuccess(result, `Full backup: ${name}`)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a mirror job, runs it, and returns the result.
|
||||
* Tracks the job and schedule for cleanup.
|
||||
* @param {Object} [settingsOverride] - Override default mirror settings
|
||||
*/
|
||||
const runMirror = async (settingsOverride = {}) => {
|
||||
const name = generateBackupJobName()
|
||||
const schedule = getDefaultSchedule()
|
||||
const config = {
|
||||
name,
|
||||
mode: 'full',
|
||||
sourceRemote: sourceRepository.id,
|
||||
remotes: { [destRepository.id]: {} },
|
||||
schedules: { '': schedule },
|
||||
settings: {
|
||||
'': {
|
||||
exportRetention: 3,
|
||||
mergeBackupsSynchronously: true,
|
||||
...settingsOverride,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const jobId = await dispatchClient.backup.createMirrorBackupJob(config)
|
||||
const job = await dispatchClient.backup.details(jobId)
|
||||
tracker.trackResource('backupJob', jobId, { name, mode: 'full', type: 'mirrorBackup' })
|
||||
|
||||
const scheduleKey = getScheduleKey(job)
|
||||
assert.ok(scheduleKey, 'Mirror schedule key is required')
|
||||
tracker.trackResource('schedule', scheduleKey, { name, backupJobId: jobId })
|
||||
|
||||
const result = await dispatchClient.backup.runMirrorJobAndGetLog(jobId, scheduleKey)
|
||||
assertBackupSuccess(result, `Mirror backup: ${name}`)
|
||||
return result
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
const setupResult = await setup()
|
||||
dispatchClient = setupResult.dispatchClient
|
||||
tracker = setupResult.tracker
|
||||
vm = setupResult.vm
|
||||
|
||||
assert.ok(vm, 'Setup should provide a test VM')
|
||||
console.log(`Using test VM: ${vm.name_label} (${vm.uuid})`)
|
||||
|
||||
// Get source backup repository
|
||||
sourceRepository = await dispatchClient.backupRepository.get({
|
||||
name: SOURCE_REPOSITORY_NAME,
|
||||
})
|
||||
|
||||
if (!sourceRepository) {
|
||||
const id = await dispatchClient.backupRepository.create(SOURCE_REPOSITORY_NAME, {
|
||||
path: process.env.BACKUP_REPOSITORY_PATH || '/tmp/xo-test-backups',
|
||||
})
|
||||
sourceRepository = await dispatchClient.backupRepository.get({ id })
|
||||
if (!sourceRepository) {
|
||||
throw new Error(`Failed to create source backup repository`)
|
||||
}
|
||||
tracker.trackResource('backupRepository', id, { name: SOURCE_REPOSITORY_NAME })
|
||||
}
|
||||
console.log(`Source repository: ${sourceRepository.name} (${sourceRepository.id})`)
|
||||
|
||||
// Get or create destination backup repository
|
||||
if (!MIRROR_DESTINATION_REPOSITORY_PATH) {
|
||||
throw new Error('MIRROR_DESTINATION_REPOSITORY_PATH environment variable is required for mirror backup tests')
|
||||
}
|
||||
|
||||
destRepository = await dispatchClient.backupRepository.get({
|
||||
name: MIRROR_DESTINATION_REPOSITORY_NAME,
|
||||
})
|
||||
|
||||
if (!destRepository) {
|
||||
const id = await dispatchClient.backupRepository.create(MIRROR_DESTINATION_REPOSITORY_NAME, {
|
||||
path: MIRROR_DESTINATION_REPOSITORY_PATH,
|
||||
})
|
||||
destRepository = await dispatchClient.backupRepository.get({ id })
|
||||
if (!destRepository) {
|
||||
throw new Error(`Failed to create destination backup repository`)
|
||||
}
|
||||
tracker.trackResource('backupRepository', id, { name: MIRROR_DESTINATION_REPOSITORY_NAME })
|
||||
}
|
||||
console.log(`Destination repository: ${destRepository.name} (${destRepository.id})`)
|
||||
|
||||
// Safety check: repositories must be empty to avoid accidental data loss
|
||||
await assertRepositoryEmpty(dispatchClient, sourceRepository)
|
||||
await assertRepositoryEmpty(dispatchClient, destRepository)
|
||||
})
|
||||
|
||||
after(async () => {
|
||||
await purgeBackupData(sourceRepository, destRepository)
|
||||
|
||||
if (dispatchClient && tracker) {
|
||||
await teardown(dispatchClient, tracker)
|
||||
}
|
||||
})
|
||||
|
||||
// =========================================================================
|
||||
// Basic Mirror Transfer
|
||||
// =========================================================================
|
||||
|
||||
describe('Basic Mirror Transfer', () => {
|
||||
after(async () => {
|
||||
await purgeBackupData(sourceRepository, destRepository)
|
||||
})
|
||||
|
||||
it('should mirror full backups from source to destination', async () => {
|
||||
console.log('Step 1: Running full backup to populate source repository...')
|
||||
const backupResult = await runFullBackup()
|
||||
assertFullOrDelta(backupResult, sourceRepository.id, { mustBeFull: true })
|
||||
|
||||
const sourceCount = await countVmBackups(sourceRepository.id, vm.uuid)
|
||||
assert.ok(sourceCount > 0, 'Source repository should contain at least one backup')
|
||||
console.log(`Source repository contains ${sourceCount} backup(s)`)
|
||||
|
||||
console.log('Step 2: Running full mirror backup (source → destination)...')
|
||||
const mirrorResult = await runMirror()
|
||||
|
||||
const transferTask = findTaskByMessage(mirrorResult, 'transfer')
|
||||
assert.ok(transferTask, 'Mirror backup should include a transfer task')
|
||||
assert.strictEqual(transferTask.status, 'success', 'Transfer task should succeed')
|
||||
|
||||
const transferredBytes = getBackupTransferredBytes(mirrorResult)
|
||||
console.log(`Mirror transferred: ${transferredBytes} bytes`)
|
||||
|
||||
const destCount = await countVmBackups(destRepository.id, vm.uuid)
|
||||
assert.ok(destCount > 0, 'Destination repository should contain backups after mirror')
|
||||
console.log(`Destination repository contains ${destCount} backup(s) after mirror`)
|
||||
})
|
||||
|
||||
it('should only transfer new backups on second mirror run', async () => {
|
||||
const destCountBefore = await countVmBackups(destRepository.id, vm.uuid)
|
||||
console.log(`Destination has ${destCountBefore} backup(s) before second mirror`)
|
||||
|
||||
console.log('Step 1: Running another full backup to add data on source...')
|
||||
await runFullBackup()
|
||||
|
||||
console.log('Step 2: Running second mirror backup...')
|
||||
await runMirror()
|
||||
|
||||
const destCountAfter = await countVmBackups(destRepository.id, vm.uuid)
|
||||
console.log(`Destination has ${destCountAfter} backup(s) after second mirror`)
|
||||
|
||||
assert.ok(
|
||||
destCountAfter > destCountBefore,
|
||||
`Destination should have more backups after second mirror (before: ${destCountBefore}, after: ${destCountAfter})`
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// =========================================================================
|
||||
// Retention — source retention > destination retention
|
||||
// =========================================================================
|
||||
|
||||
describe('Retention: source > destination', () => {
|
||||
after(async () => {
|
||||
await purgeBackupData(sourceRepository, destRepository)
|
||||
})
|
||||
|
||||
it('should cap destination backups to destination exportRetention', async () => {
|
||||
const SOURCE_RETENTION = 4
|
||||
const DEST_RETENTION = 2
|
||||
|
||||
// Populate source with SOURCE_RETENTION full backups
|
||||
console.log(`Creating ${SOURCE_RETENTION} full backups on source (retention=${SOURCE_RETENTION})...`)
|
||||
for (let i = 0; i < SOURCE_RETENTION; i++) {
|
||||
console.log(` Backup ${i + 1}/${SOURCE_RETENTION}...`)
|
||||
await runFullBackup()
|
||||
}
|
||||
|
||||
const sourceCount = await countVmBackups(sourceRepository.id, vm.uuid)
|
||||
console.log(`Source has ${sourceCount} backup(s)`)
|
||||
assert.ok(sourceCount >= SOURCE_RETENTION, `Source should have at least ${SOURCE_RETENTION} backups`)
|
||||
|
||||
// Mirror with lower retention on destination
|
||||
console.log(`Running mirror with destination exportRetention=${DEST_RETENTION}...`)
|
||||
await runMirror({ exportRetention: DEST_RETENTION })
|
||||
|
||||
const destCount = await countVmBackups(destRepository.id, vm.uuid)
|
||||
console.log(`Destination has ${destCount} backup(s) (expected ≤ ${DEST_RETENTION})`)
|
||||
|
||||
assert.ok(
|
||||
destCount <= DEST_RETENTION,
|
||||
`Destination should have at most ${DEST_RETENTION} backups (got ${destCount}) ` +
|
||||
`— source has more backups but destination retention should cap it`
|
||||
)
|
||||
assert.ok(destCount > 0, 'Destination should have at least 1 backup')
|
||||
})
|
||||
})
|
||||
|
||||
// =========================================================================
|
||||
// Retention — destination retention > source retention
|
||||
// =========================================================================
|
||||
|
||||
describe('Retention: destination > source', () => {
|
||||
after(async () => {
|
||||
await purgeBackupData(sourceRepository, destRepository)
|
||||
})
|
||||
|
||||
it('should accumulate backups on destination up to its retention', async () => {
|
||||
const SOURCE_RETENTION = 2
|
||||
const DEST_RETENTION = 4
|
||||
|
||||
// Create SOURCE_RETENTION backups, mirror, repeat — destination should accumulate
|
||||
|
||||
// Round 1: populate source + mirror
|
||||
console.log(`Round 1: creating ${SOURCE_RETENTION} backups on source...`)
|
||||
for (let i = 0; i < SOURCE_RETENTION; i++) {
|
||||
await runFullBackup()
|
||||
}
|
||||
|
||||
console.log(`Round 1: mirroring with destination exportRetention=${DEST_RETENTION}...`)
|
||||
await runMirror({ exportRetention: DEST_RETENTION })
|
||||
|
||||
const destCountRound1 = await countVmBackups(destRepository.id, vm.uuid)
|
||||
console.log(`Destination has ${destCountRound1} backup(s) after round 1`)
|
||||
assert.ok(destCountRound1 > 0, 'Destination should have backups after first mirror')
|
||||
|
||||
// Round 2: create more backups on source + mirror again
|
||||
console.log(`Round 2: creating ${SOURCE_RETENTION} more backups on source...`)
|
||||
for (let i = 0; i < SOURCE_RETENTION; i++) {
|
||||
await runFullBackup()
|
||||
}
|
||||
|
||||
console.log(`Round 2: mirroring again...`)
|
||||
await runMirror({ exportRetention: DEST_RETENTION })
|
||||
|
||||
const destCountRound2 = await countVmBackups(destRepository.id, vm.uuid)
|
||||
console.log(`Destination has ${destCountRound2} backup(s) after round 2`)
|
||||
|
||||
// Destination should have accumulated more backups than source currently holds
|
||||
const sourceCount = await countVmBackups(sourceRepository.id, vm.uuid)
|
||||
console.log(`Source currently has ${sourceCount} backup(s)`)
|
||||
|
||||
assert.ok(
|
||||
destCountRound2 > destCountRound1,
|
||||
`Destination should accumulate backups across mirror runs ` +
|
||||
`(round 1: ${destCountRound1}, round 2: ${destCountRound2})`
|
||||
)
|
||||
|
||||
assert.ok(
|
||||
destCountRound2 <= DEST_RETENTION,
|
||||
`Destination should not exceed its retention of ${DEST_RETENTION} (got ${destCountRound2})`
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -4,14 +4,22 @@ import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
import { backupConfig } from '../backup.config.js'
|
||||
import { assertFullOrDelta, generateBackupJobName, getDefaultSchedule, getScheduleKey } from '../utils/index.js'
|
||||
import {
|
||||
assertFullOrDelta,
|
||||
assertFullOrDeltaForSr,
|
||||
findTaskByMessage,
|
||||
generateBackupJobName,
|
||||
getBackupTransferredBytes,
|
||||
getDefaultSchedule,
|
||||
getScheduleKey,
|
||||
} from '../utils/index.js'
|
||||
import {
|
||||
generateExportFileName,
|
||||
validateVhdIntegrity,
|
||||
validateXvaIntegrity,
|
||||
formatBytes,
|
||||
} from '../utils/exportUtils.js'
|
||||
import { assertBackupSuccess } from '../utils/backupUtils.js'
|
||||
import { assertBackupSuccess, assertRepositoryEmpty } from '../utils/backupUtils.js'
|
||||
import { setup, teardown } from './setup.js'
|
||||
|
||||
/** Default backup repository name used across all tests */
|
||||
@@ -125,6 +133,9 @@ describe('Backup + Replication Combined Tests', () => {
|
||||
})
|
||||
}
|
||||
|
||||
// Safety check: repository must be empty to avoid accidental data loss
|
||||
await assertRepositoryEmpty(dispatchClient, backupRepository)
|
||||
|
||||
// Get SR for health checks and restorations
|
||||
const srId = process.env.SR_ID
|
||||
if (!srId) {
|
||||
@@ -443,6 +454,123 @@ describe('Backup + Replication Combined Tests', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// =========================================================================
|
||||
// CR Mode - Delta Replication to SR (target VM reuse + snapshots)
|
||||
// =========================================================================
|
||||
|
||||
describe('CR Mode - Delta Replication to SR', () => {
|
||||
it('should replicate VM to SR, reuse target VM on second run, and create snapshots', async () => {
|
||||
const targetSrUuid = sr.uuid
|
||||
|
||||
// Create CR mode backup job: delta mode + srs (no remotes)
|
||||
const name = generateBackupJobName()
|
||||
const schedule = getDefaultSchedule()
|
||||
const config = {
|
||||
name,
|
||||
mode: 'delta',
|
||||
schedules: { '': schedule },
|
||||
settings: { '': { timezone: 'Europe/Paris', copyRetention: 3, mergeBackupsSynchronously: true } },
|
||||
vms: { [vm.uuid]: vm },
|
||||
srs: { [targetSrUuid]: true },
|
||||
}
|
||||
|
||||
const jobId = await dispatchClient.backup.createBackupJob(config)
|
||||
const job = await dispatchClient.backup.details(jobId)
|
||||
assert.strictEqual(job.mode, 'delta', 'CR mode job should be delta')
|
||||
|
||||
tracker.trackResource('backupJob', jobId, { name, mode: 'delta' })
|
||||
|
||||
const scheduleKey = getScheduleKey(job)
|
||||
assert.ok(scheduleKey, 'Schedule key is required')
|
||||
tracker.trackResource('schedule', scheduleKey, { name, backupJobId: jobId })
|
||||
|
||||
// Capture VM list before first replication to detect the new target VM
|
||||
const vmsBefore = await dispatchClient.vm.list()
|
||||
const vmUuidsBefore = new Set(vmsBefore.map(v => v.uuid))
|
||||
|
||||
// --- First replication (full — no existing target) ---
|
||||
console.log('Running first CR replication (expected full)...')
|
||||
const result1 = await dispatchClient.backup.runJobAndGetLog(jobId, scheduleKey)
|
||||
assertBackupSuccess(result1, 'First CR replication')
|
||||
assertFullOrDeltaForSr(result1, targetSrUuid, { mustBeFull: true })
|
||||
|
||||
// Verify "target snapshot" task exists in the log
|
||||
const snapshotTask1 = findTaskByMessage(result1, 'target snapshot')
|
||||
assert.ok(snapshotTask1, 'First replication should include a "target snapshot" task')
|
||||
assert.strictEqual(snapshotTask1.status, 'success', 'Target snapshot task should succeed')
|
||||
|
||||
// Find the newly created target VM
|
||||
const vmsAfterFirst = await dispatchClient.vm.list()
|
||||
const newVms = vmsAfterFirst.filter(v => !vmUuidsBefore.has(v.uuid))
|
||||
assert.strictEqual(newVms.length, 1, 'First replication should create exactly one new target VM')
|
||||
|
||||
const targetVmUuid = newVms[0].uuid
|
||||
restoredVmUuids.push(targetVmUuid)
|
||||
console.log(`Target VM created: ${newVms[0].name_label} (${targetVmUuid})`)
|
||||
|
||||
// Check target VM has at least one snapshot
|
||||
const targetVmAfterFirst = await dispatchClient.vm.details(targetVmUuid)
|
||||
assert.ok(targetVmAfterFirst, 'Target VM should exist')
|
||||
const snapshotsAfterFirst = targetVmAfterFirst.snapshots?.length ?? 0
|
||||
assert.ok(
|
||||
snapshotsAfterFirst >= 1,
|
||||
`Target VM should have ≥1 snapshot after first replication, got ${snapshotsAfterFirst}`
|
||||
)
|
||||
console.log(`Target VM has ${snapshotsAfterFirst} snapshot(s) after first replication`)
|
||||
|
||||
// Record first transfer size for efficiency comparison
|
||||
const firstTransferSize = getBackupTransferredBytes(result1)
|
||||
console.log(`First replication transferred: ${firstTransferSize} bytes`)
|
||||
|
||||
// --- Second replication (delta — should reuse target VM) ---
|
||||
console.log('Running second CR replication (expected delta)...')
|
||||
const result2 = await dispatchClient.backup.runJobAndGetLog(jobId, scheduleKey)
|
||||
assertBackupSuccess(result2, 'Second CR replication')
|
||||
assertFullOrDeltaForSr(result2, targetSrUuid, { mustBeFull: false })
|
||||
|
||||
// Verify "target snapshot" task in second run
|
||||
const snapshotTask2 = findTaskByMessage(result2, 'target snapshot')
|
||||
assert.ok(snapshotTask2, 'Second replication should include a "target snapshot" task')
|
||||
assert.strictEqual(snapshotTask2.status, 'success', 'Target snapshot task should succeed on second run')
|
||||
|
||||
// Verify NO new VMs were created (target VM was reused)
|
||||
const vmsAfterSecond = await dispatchClient.vm.list()
|
||||
const newVmsAfterSecond = vmsAfterSecond.filter(v => !vmUuidsBefore.has(v.uuid))
|
||||
assert.strictEqual(
|
||||
newVmsAfterSecond.length,
|
||||
1,
|
||||
`Second replication should NOT create a new VM — expected 1 new VM total, got ${newVmsAfterSecond.length}`
|
||||
)
|
||||
|
||||
// Target VM should now have more snapshots
|
||||
const targetVmAfterSecond = await dispatchClient.vm.details(targetVmUuid)
|
||||
const snapshotsAfterSecond = targetVmAfterSecond.snapshots?.length ?? 0
|
||||
assert.ok(
|
||||
snapshotsAfterSecond > snapshotsAfterFirst,
|
||||
`Target VM should have more snapshots after second replication ` +
|
||||
`(before: ${snapshotsAfterFirst}, after: ${snapshotsAfterSecond})`
|
||||
)
|
||||
console.log(`Target VM has ${snapshotsAfterSecond} snapshot(s) after second replication`)
|
||||
|
||||
// Verify delta efficiency: second transfer should be ≤ first
|
||||
const secondTransferSize = getBackupTransferredBytes(result2)
|
||||
console.log(`Second replication transferred: ${secondTransferSize} bytes`)
|
||||
|
||||
if (firstTransferSize !== null && secondTransferSize !== null) {
|
||||
assert.ok(
|
||||
secondTransferSize <= firstTransferSize,
|
||||
`Delta transfer (${secondTransferSize} bytes) should be ≤ full transfer (${firstTransferSize} bytes)`
|
||||
)
|
||||
}
|
||||
|
||||
console.log(
|
||||
`CR mode test passed: target VM reused (${targetVmUuid}), ` +
|
||||
`snapshots ${snapshotsAfterFirst}→${snapshotsAfterSecond}, ` +
|
||||
`transfer ${firstTransferSize}→${secondTransferSize} bytes`
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// =========================================================================
|
||||
// State Validation Between Operations
|
||||
// =========================================================================
|
||||
|
||||
@@ -1,5 +1,29 @@
|
||||
import assert from 'node:assert'
|
||||
|
||||
/**
|
||||
* Asserts that a backup repository is empty (contains no VM backups).
|
||||
*
|
||||
* This safety check prevents tests from running against repositories that
|
||||
* already contain data, which could lead to accidental data loss if the
|
||||
* repository is misconfigured to point at a production remote.
|
||||
*
|
||||
* @param {import('../client/dispatchClient.js').DispatchClient} dispatchClient
|
||||
* @param {{id: string, name: string}} repository - The backup repository to check
|
||||
* @throws {Error} If the repository contains existing backups
|
||||
*/
|
||||
export const assertRepositoryEmpty = async (dispatchClient, repository) => {
|
||||
const backups = await dispatchClient.backup.listVmBackups([repository.id])
|
||||
const count = Object.values(backups).flatMap(vmBackups => Object.values(vmBackups).flat()).length
|
||||
|
||||
if (count > 0) {
|
||||
throw new Error(
|
||||
`Repository "${repository.name}" (${repository.id}) contains ${count} existing backup(s). ` +
|
||||
`Tests require empty repositories to avoid accidental data loss. ` +
|
||||
`Please purge the backups manually or use a dedicated test repository.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts an error message from a backup task's result or error fields.
|
||||
*
|
||||
|
||||
@@ -196,6 +196,116 @@ export function getIsFullForBackupRepository(logEntry, backupRepositoryUuid) {
|
||||
return undefined // Export operation not found for this backup repository
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SR REPLICATION (CR/DR MODE) UTILITIES
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Determines whether a replication was full or incremental for a specific SR.
|
||||
*
|
||||
* Similar to getIsFullForBackupRepository but checks for SR-type exports
|
||||
* (used in CR mode — delta replication to SR).
|
||||
*
|
||||
* @param {Object} logEntry - Backup log entry object from XenOrchestra
|
||||
* @param {string} srUuid - UUID of the target SR
|
||||
* @returns {boolean|undefined} True if full, false if incremental, undefined if not found
|
||||
*/
|
||||
export function getIsFullForSr(logEntry, srUuid) {
|
||||
if (!logEntry || typeof logEntry !== 'object') {
|
||||
throw new Error('Valid log entry object is required')
|
||||
}
|
||||
|
||||
assertNonEmptyString(srUuid, 'Valid SR UUID string is required')
|
||||
|
||||
function searchTasks(tasks) {
|
||||
if (!Array.isArray(tasks)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
for (const task of tasks) {
|
||||
if (
|
||||
task.message === 'export' &&
|
||||
task.data &&
|
||||
task.data.id === srUuid &&
|
||||
task.data.type === 'SR' &&
|
||||
typeof task.data.isFull === 'boolean'
|
||||
) {
|
||||
return task.data.isFull
|
||||
}
|
||||
|
||||
if (task.tasks && task.tasks.length > 0) {
|
||||
const result = searchTasks(task.tasks)
|
||||
if (result !== undefined) {
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (logEntry.tasks && logEntry.tasks.length > 0) {
|
||||
const result = searchTasks(logEntry.tasks)
|
||||
if (result !== undefined) {
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts replication type (full or delta) for a given target SR.
|
||||
*
|
||||
* @param {Object} result - Backup log result
|
||||
* @param {string} srUuid - UUID of the target SR
|
||||
* @param {Object} options
|
||||
* @param {boolean} options.mustBeFull - Whether replication should be full
|
||||
* @throws {Error} If replication type does not match expectation
|
||||
*/
|
||||
export function assertFullOrDeltaForSr(result, srUuid, { mustBeFull }) {
|
||||
const isFull = getIsFullForSr(result, srUuid)
|
||||
assert.strictEqual(
|
||||
isFull,
|
||||
mustBeFull,
|
||||
`SR replication should be ${mustBeFull ? 'full' : 'delta'}, got ${isFull === undefined ? 'not found' : isFull ? 'full' : 'delta'}`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the first task matching a given message name in a backup log entry.
|
||||
*
|
||||
* Recursively searches through all tasks and subtasks.
|
||||
*
|
||||
* @param {Object} logEntry - Backup log entry from XenOrchestra
|
||||
* @param {string} message - Task message to search for (e.g., 'target snapshot', 'transfer')
|
||||
* @returns {Object|null} The matching task or null if not found
|
||||
*/
|
||||
export function findTaskByMessage(logEntry, message) {
|
||||
function search(tasks) {
|
||||
if (!Array.isArray(tasks)) {
|
||||
return null
|
||||
}
|
||||
|
||||
for (const task of tasks) {
|
||||
if (task.message === message) {
|
||||
return task
|
||||
}
|
||||
|
||||
if (task.tasks && task.tasks.length > 0) {
|
||||
const result = search(task.tasks)
|
||||
if (result) {
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
return search(logEntry.tasks || [])
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BACKUP ASSERTION UTILITIES
|
||||
// =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user