mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-09-10 22:14:48 -05:00
fix(xo-server/v2v): incorrect uuid for running VM without snapshot (#10194)
* add more details in task log * fix an issue with VM with more than 10 snapshot, add unit test for this parsing * fix the real bug: vmware do not update the vmdk description file before the next VM reboot. It means that the uid of the snapshot and vm are the same, thus finishing the transfer with " nothing to transfer" . We are now using the path on source as key + add unit test + add compatibiliy with legacy VM migrated with the uid. * add e2e test for v2v
This commit is contained in:
Florent BEAUCHAMP
committed by
GitHub
GitHub
parent
9c6d10327f
commit
be94d0d083
@@ -67,6 +67,38 @@ SDN_CONTROLLER_VM_ID=
|
||||
# Must be a test-scoped path (containing 'test', 'qa', or 'tmp/') for cleanup safety
|
||||
VHD_EXPORT_PATH=/tmp/xo-test-exports
|
||||
|
||||
# ===========================================
|
||||
# ESXI MIGRATION (V2V) TEST CONFIGURATION
|
||||
# ===========================================
|
||||
|
||||
# `yarn qa:import:esxi` is skipped unless all four of these are set.
|
||||
# The source VM must be RUNNING and have NO SNAPSHOT: the test migrates a live VM and checks
|
||||
# that XO takes its own snapshot. It leaves the VM halted, with that snapshot — restart is
|
||||
# automatic, but the snapshot has to be removed by hand before the next run.
|
||||
# ESXI_VM_ID is the VMware MoRef id (a number, as shown by esxi.listVms), not a UUID.
|
||||
ESXI_HOST=
|
||||
ESXI_USER=root
|
||||
ESXI_PASSWORD=
|
||||
ESXI_VM_ID=
|
||||
|
||||
# Set to "true" to validate the ESXi certificate — off by default, lab hosts are self-signed
|
||||
ESXI_SSL_VERIFY=false
|
||||
|
||||
# The target SR is SR_ID above. Network and template are looked up on that SR's pool:
|
||||
# the first network with a PIF, and a template named ESXI_TEMPLATE_NAME. Override either
|
||||
# with an explicit UUID when the guess is wrong.
|
||||
ESXI_NETWORK_ID=
|
||||
ESXI_TEMPLATE_ID=
|
||||
ESXI_TEMPLATE_NAME=Other install media
|
||||
|
||||
# Per-request timeout for the migrations and the raw disk comparison (default 4 h)
|
||||
ESXI_TEST_TIMEOUT_MS=
|
||||
|
||||
# Between the two transfers the source guest is hard reset so it writes to its active disk,
|
||||
# giving the delta pass something real to move. This is how long it is then given to boot and
|
||||
# flush — raise it if the test reports an empty delta (default 90 s)
|
||||
ESXI_RESET_SETTLE_MS=
|
||||
|
||||
# Gmail: enable 2-Step Verification then generate an App Password at
|
||||
# https://myaccount.google.com/apppasswords
|
||||
SMTP_HOST=smtp.gmail.com
|
||||
|
||||
@@ -21,9 +21,11 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@vates/read-chunk": "^1.2.1",
|
||||
"@xen-orchestra/backups": "^0.73.9",
|
||||
"@xen-orchestra/fs": "^4.9.3",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"@xen-orchestra/vmware-explorer": "^0.14.0",
|
||||
"nodemailer": "^7.0.13",
|
||||
"xo-lib": "^0.11.2"
|
||||
},
|
||||
@@ -48,6 +50,7 @@
|
||||
"report": "node --env-file-if-exists=.env scripts/run-and-report.mjs",
|
||||
"report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test",
|
||||
"qa:backup:file-restore": "node --env-file-if-exists=.env --test tests/backup.file-restore.test.js",
|
||||
"qa:import:esxi": "node --env-file-if-exists=.env --test --test-force-exit tests/import.esxi.test.js",
|
||||
"qa:load:backup:delta": "node --env-file-if-exists=.env scripts/backup-load-delta.mjs"
|
||||
}
|
||||
}
|
||||
|
||||
355
@xen-orchestra/qa-test/tests/import.esxi.test.js
Normal file
355
@xen-orchestra/qa-test/tests/import.esxi.test.js
Normal file
@@ -0,0 +1,355 @@
|
||||
// installs the transports, including the one capturing every debug record to a file in
|
||||
// os.tmpdir() — must come first so nothing is logged before it is configured
|
||||
import '../logSetup.js'
|
||||
|
||||
import assert from 'node:assert/strict'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { DispatchClient } from '../client/dispatchClient.js'
|
||||
import { createResourceTracker } from '../utils/resourceTracker.js'
|
||||
import { getRequiredEnv } from '../utils/index.js'
|
||||
import { formatBytes } from '../utils/exportUtils.js'
|
||||
import {
|
||||
assertRawExportsAreIdentical,
|
||||
assertSourceIsRunningWithoutSnapshot,
|
||||
assertTransferSequence,
|
||||
churnSourceByReset,
|
||||
connectToEsxi,
|
||||
getAndLogSourceState,
|
||||
getEsxiConfig,
|
||||
getEsxiSkipReason,
|
||||
getOrderedDisks,
|
||||
optionalEnv,
|
||||
resetSourceState,
|
||||
waitForBlockedOperations,
|
||||
waitForSourceSnapshot,
|
||||
} from '../utils/esxiMigrationUtils.js'
|
||||
|
||||
const log = createLogger('qa:import:esxi')
|
||||
|
||||
const XO_SNAPSHOT_NAME = '[V2V] migration to XCP-ng'
|
||||
|
||||
describe('ESXi migration (V2V)', { skip: getEsxiSkipReason() }, () => {
|
||||
/** @type {import('../client/dispatchClient.js').DispatchClient} */
|
||||
let dispatchClient
|
||||
/** @type {Object} */
|
||||
let tracker
|
||||
/** @type {import('@xen-orchestra/vmware-explorer/esxi.mjs').default} */
|
||||
let esxi
|
||||
/** @type {ReturnType<typeof getEsxiConfig>} */
|
||||
let esxiConfig
|
||||
/** @type {Object} */
|
||||
let sourceMetadata
|
||||
/** Parameters shared by every `vm.importFromEsxi` call */
|
||||
let migrationParams
|
||||
|
||||
before(async () => {
|
||||
esxiConfig = getEsxiConfig()
|
||||
|
||||
dispatchClient = new DispatchClient()
|
||||
await dispatchClient.initialize()
|
||||
tracker = createResourceTracker()
|
||||
|
||||
esxi = await connectToEsxi(esxiConfig)
|
||||
sourceMetadata = await assertSourceIsRunningWithoutSnapshot(esxi, esxiConfig.vmId)
|
||||
|
||||
const sr = await dispatchClient.sr.details(getRequiredEnv('SR_ID'))
|
||||
assert.notEqual(sr, undefined, 'SR_ID does not resolve to an SR')
|
||||
|
||||
migrationParams = {
|
||||
host: esxiConfig.host,
|
||||
user: esxiConfig.user,
|
||||
password: esxiConfig.password,
|
||||
sslVerify: esxiConfig.sslVerify,
|
||||
vm: esxiConfig.vmId,
|
||||
sr: sr.uuid,
|
||||
network: await findNetwork(dispatchClient, sr.$pool),
|
||||
template: await findTemplate(dispatchClient, sr.$pool),
|
||||
}
|
||||
|
||||
log.info('Ready to migrate source VM', {
|
||||
name: sourceMetadata.name_label,
|
||||
vmId: esxiConfig.vmId,
|
||||
disks: sourceMetadata.disks.length,
|
||||
sr: sr.name_label,
|
||||
})
|
||||
log.debug('Migration parameters', { ...migrationParams, password: '<redacted>' })
|
||||
log.debug('Resolved timings', {
|
||||
requestTimeoutMinutes: Math.round(esxiConfig.timeout / 60_000),
|
||||
resetSettleSeconds: Math.round(esxiConfig.resetSettleMs / 1000),
|
||||
})
|
||||
log.debug('Target pool objects resolved', {
|
||||
pool: sr.$pool,
|
||||
sr: { uuid: sr.uuid, name_label: sr.name_label },
|
||||
network: migrationParams.network,
|
||||
template: migrationParams.template,
|
||||
})
|
||||
})
|
||||
|
||||
after(async () => {
|
||||
// put the source back to running without snapshots, so the suite can be run again
|
||||
// without touching ESXi by hand
|
||||
if (esxi !== undefined) {
|
||||
try {
|
||||
await resetSourceState(esxi, esxiConfig.vmId)
|
||||
} catch (error) {
|
||||
log.warn(`Failed to reset the source VM, remove its "${XO_SNAPSHOT_NAME}" snapshot by hand`, { error })
|
||||
}
|
||||
}
|
||||
|
||||
if (tracker !== undefined) {
|
||||
const vmIds = tracker.getTrackedResourcesByType('vm').map(({ id }) => id)
|
||||
if (vmIds.length > 0) {
|
||||
try {
|
||||
await dispatchClient.cleanup.fullCleanup({
|
||||
cleanupVMs: true,
|
||||
cleanupBackupJobs: false,
|
||||
cleanupSchedules: false,
|
||||
additionalVmIds: vmIds,
|
||||
})
|
||||
} catch (error) {
|
||||
log.warn('Failed to delete the migrated VMs', { vmIds, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await dispatchClient?.close()
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// The scenario XO users actually run: one call, source running, no snapshot yet. XO
|
||||
// snapshots the source, transfers everything up to that snapshot while the guest keeps
|
||||
// running, then stops it and transfers what was written since.
|
||||
//
|
||||
// What makes this the scenario that covers the regression: `migrationfromEsxi` reads the
|
||||
// source metadata once, right after taking its snapshot, and `#importDisks` uses that same
|
||||
// chain for both passes. At that instant the delta still carries the CID of its parent, so
|
||||
// the delta pass always sees the two disks with an identical CID — whatever the guest wrote
|
||||
// in between. Matching disks on their CID made the delta look already imported and skipped
|
||||
// it silently. Re-reading the metadata later, as a second migration would, hides this: by
|
||||
// then VMware has rewritten the CID of any disk the guest touched.
|
||||
//
|
||||
// The guest is rebooted while the base transfers, so the delta holds real data and the byte
|
||||
// comparison in step 3 has something to chew on.
|
||||
//
|
||||
// step 1: one call, stopSource → base + delta, VM startable afterwards
|
||||
// step 2: same, now halted source → new VM, whole chain in one pass
|
||||
// step 3: step 1 and step 2 disks must be byte for byte identical
|
||||
// step 4: from a clean source again, the resume path and its unstartable partial VM
|
||||
//
|
||||
// Kept in a single test because every step depends on the state left by the previous one:
|
||||
// splitting them would report several misleading failures instead of the one that matters.
|
||||
// -----------------------------------------------------------------------------
|
||||
it('rebuilds the same disks from a snapshot plus its delta as from a single full transfer', async () => {
|
||||
// --- Step 1/4: one migration, with the guest writing while the base transfers ---
|
||||
log.info('Step 1/4: migrating a running source in one call, churning the guest meanwhile')
|
||||
const migration = migrate({ stopSource: true })
|
||||
let migrationSettled = false
|
||||
migration.then(
|
||||
() => {
|
||||
migrationSettled = true
|
||||
},
|
||||
() => {
|
||||
migrationSettled = true
|
||||
}
|
||||
)
|
||||
|
||||
// churn failures must not fail the run: the delta is then only what the guest happened to
|
||||
// write, which still exercises the regression, just with less data
|
||||
const churning = (async () => {
|
||||
await waitForSourceSnapshot(esxi, esxiConfig.vmId)
|
||||
if (migrationSettled) {
|
||||
log.warn('Migration finished before the guest could be churned, the delta will be small')
|
||||
return
|
||||
}
|
||||
await churnSourceByReset(esxi, esxiConfig.vmId, { settleMs: esxiConfig.resetSettleMs })
|
||||
})().catch(error => log.warn('Could not churn the source during the transfer', { error }))
|
||||
|
||||
const twoStepVmUuid = await migration
|
||||
await churning
|
||||
tracker.trackResource('vm', twoStepVmUuid, { name: 'V2V two step' })
|
||||
|
||||
const sourceAfter = await getAndLogSourceState(esxi, esxiConfig.vmId, 'after the two step migration')
|
||||
assert.equal(sourceAfter.powerState, 'poweredOff', 'the source should have been stopped before the last transfer')
|
||||
assert.equal(
|
||||
sourceAfter.snapshots?.snapshots.find(({ uid }) => uid === sourceAfter.snapshots.current)?.displayName,
|
||||
XO_SNAPSHOT_NAME,
|
||||
'the current source snapshot should be the one XO took for the migration'
|
||||
)
|
||||
|
||||
const twoStepDisks = await getOrderedDisks(dispatchClient, twoStepVmUuid)
|
||||
assert.equal(twoStepDisks.length, sourceMetadata.disks.length, 'every source disk should have been created')
|
||||
for (const disk of twoStepDisks) {
|
||||
// a single `base` is the regression: the delta pass decided there was nothing to import.
|
||||
// A second `base` would mean the whole disk was transferred again instead of a delta
|
||||
const [base, delta] = assertTransferSequence(disk, ['base', 'snapshot'])
|
||||
assert.ok(
|
||||
delta.megabytes <= base.megabytes,
|
||||
`the delta of disk ${disk.position} should not be larger than the base transfer ` +
|
||||
`(delta ${delta.megabytes} MB, base ${base.megabytes} MB)`
|
||||
)
|
||||
log.info('Disk transferred in two passes', {
|
||||
position: disk.position,
|
||||
vdi: disk.uuid,
|
||||
baseMB: base.megabytes,
|
||||
deltaMB: delta.megabytes,
|
||||
})
|
||||
if (delta.megabytes === 0) {
|
||||
// not a failure: the sequence assertion above already proves the delta pass ran. But
|
||||
// step 3 only compares zeroes then, so the run is weaker than it looks
|
||||
log.warn('Empty delta, the raw comparison will not exercise much', {
|
||||
position: disk.position,
|
||||
hint: 'raise ESXI_RESET_SETTLE_MS so the guest has time to write after the reset',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await waitForBlockedOperations(dispatchClient, twoStepVmUuid, { blocked: false })
|
||||
|
||||
// --- Step 2/4: migrate the now halted source in one pass ---------------------
|
||||
// the source has not been touched since, so this VM must hold exactly the same content
|
||||
log.info('Step 2/4: migrating the halted source')
|
||||
const fullTransferVmUuid = await migrate({ stopSource: true })
|
||||
tracker.trackResource('vm', fullTransferVmUuid, { name: 'V2V full transfer' })
|
||||
|
||||
assert.notEqual(
|
||||
fullTransferVmUuid,
|
||||
twoStepVmUuid,
|
||||
'migrating a completed source again should create a new VM, not reuse the previous one'
|
||||
)
|
||||
|
||||
await waitForBlockedOperations(dispatchClient, fullTransferVmUuid, { blocked: false })
|
||||
|
||||
const fullTransferDisks = await getOrderedDisks(dispatchClient, fullTransferVmUuid)
|
||||
assert.equal(fullTransferDisks.length, twoStepDisks.length, 'both VMs should have the same number of disks')
|
||||
for (const disk of fullTransferDisks) {
|
||||
assertTransferSequence(disk, ['base'])
|
||||
}
|
||||
|
||||
// --- Step 3/4: both reconstructions must be identical ------------------------
|
||||
log.info('Step 3/4: comparing the raw exports')
|
||||
for (const [index, twoStepDisk] of twoStepDisks.entries()) {
|
||||
const fullTransferDisk = fullTransferDisks[index]
|
||||
|
||||
assert.equal(
|
||||
twoStepDisk.size,
|
||||
fullTransferDisk.size,
|
||||
`disk ${twoStepDisk.position} should have the same virtual size in both VMs`
|
||||
)
|
||||
|
||||
log.info('Comparing raw exports', {
|
||||
position: twoStepDisk.position,
|
||||
size: formatBytes(twoStepDisk.size),
|
||||
twoStep: twoStepDisk.uuid,
|
||||
fullTransfer: fullTransferDisk.uuid,
|
||||
})
|
||||
|
||||
await assertRawExportsAreIdentical(dispatchClient.restApiClient, {
|
||||
referenceVdi: fullTransferDisk.uuid,
|
||||
candidateVdi: twoStepDisk.uuid,
|
||||
size: fullTransferDisk.size,
|
||||
timeout: esxiConfig.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
// --- Step 4/4: the resume path, from a clean source again --------------------
|
||||
// a migration that leaves the source running produces a VM holding only the data up to
|
||||
// the snapshot: it must not be bootable, and a later run must continue it rather than
|
||||
// start over. This needs a snapshot free source, hence the reset
|
||||
log.info('Step 4/4: resuming an interrupted migration, from a clean source')
|
||||
await resetSourceState(esxi, esxiConfig.vmId)
|
||||
await getAndLogSourceState(esxi, esxiConfig.vmId, 'before the resume scenario')
|
||||
|
||||
const partialVmUuid = await migrate({ stopSource: false })
|
||||
tracker.trackResource('vm', partialVmUuid, { name: 'V2V resume' })
|
||||
|
||||
const partialDisks = await getOrderedDisks(dispatchClient, partialVmUuid)
|
||||
assert.equal(partialDisks.length, sourceMetadata.disks.length, 'every source disk should have been created')
|
||||
for (const disk of partialDisks) {
|
||||
assertTransferSequence(disk, ['base'])
|
||||
}
|
||||
|
||||
await waitForBlockedOperations(dispatchClient, partialVmUuid, { blocked: true })
|
||||
await assert.rejects(
|
||||
() => dispatchClient.vm.start(partialVmUuid),
|
||||
'starting a partially transferred VM should be refused'
|
||||
)
|
||||
|
||||
const resumedVmUuid = await migrate({ stopSource: true })
|
||||
assert.equal(
|
||||
resumedVmUuid,
|
||||
partialVmUuid,
|
||||
'the second migration should resume the VM left by the first one, not create another'
|
||||
)
|
||||
|
||||
for (const disk of await getOrderedDisks(dispatchClient, resumedVmUuid)) {
|
||||
assertTransferSequence(disk, ['base', 'snapshot'])
|
||||
}
|
||||
await waitForBlockedOperations(dispatchClient, resumedVmUuid, { blocked: false })
|
||||
})
|
||||
|
||||
/**
|
||||
* Runs a migration of the configured source VM and returns the UUID of the XCP-ng VM.
|
||||
* @private
|
||||
*/
|
||||
async function migrate({ stopSource }) {
|
||||
const startTime = Date.now()
|
||||
log.debug('Calling vm.importFromEsxi', { vm: esxiConfig.vmId, stopSource })
|
||||
|
||||
let result
|
||||
try {
|
||||
result = await dispatchClient.xoClient.call('vm.importFromEsxi', { ...migrationParams, stopSource })
|
||||
} catch (cause) {
|
||||
// `succeeded` maps the source VM to the XCP-ng VM created before the failure, and is
|
||||
// the only way to find the leftover VM to clean up
|
||||
log.warn('Migration failed', { stopSource, succeeded: cause.succeeded, error: cause })
|
||||
throw cause
|
||||
}
|
||||
|
||||
const vmUuid = result[esxiConfig.vmId]
|
||||
log.debug('vm.importFromEsxi returned', { result })
|
||||
|
||||
assert.equal(typeof vmUuid, 'string', `vm.importFromEsxi did not return a VM for ${esxiConfig.vmId}`)
|
||||
log.info('Migration done', { stopSource, vmUuid, durationSeconds: Math.round((Date.now() - startTime) / 1000) })
|
||||
|
||||
return vmUuid
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Picks a network the migrated VM can be attached to, on the same pool as the target SR.
|
||||
* @private
|
||||
*/
|
||||
async function findNetwork(dispatchClient, poolId) {
|
||||
const override = optionalEnv('ESXI_NETWORK_ID')
|
||||
if (override !== undefined) {
|
||||
return override
|
||||
}
|
||||
|
||||
const networks = await dispatchClient.restApiClient.getObjects('/rest/v0/networks')
|
||||
// a network without PIF is not connected to anything physical
|
||||
const network = networks.find(candidate => candidate.$pool === poolId && candidate.PIFs?.length > 0)
|
||||
|
||||
assert.notEqual(network, undefined, `no network with a PIF found on pool ${poolId} — set ESXI_NETWORK_ID`)
|
||||
return network.uuid
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks the template the migrated VM is built from, on the same pool as the target SR.
|
||||
* @private
|
||||
*/
|
||||
async function findTemplate(dispatchClient, poolId) {
|
||||
const override = optionalEnv('ESXI_TEMPLATE_ID')
|
||||
if (override !== undefined) {
|
||||
return override
|
||||
}
|
||||
|
||||
const name = optionalEnv('ESXI_TEMPLATE_NAME') ?? 'Other install media'
|
||||
const templates = await dispatchClient.restApiClient.getObjects('/rest/v0/vm-templates')
|
||||
const template = templates.find(candidate => candidate.$pool === poolId && candidate.name_label === name)
|
||||
|
||||
assert.notEqual(template, undefined, `template "${name}" not found on pool ${poolId} — set ESXI_TEMPLATE_ID`)
|
||||
return template.uuid
|
||||
}
|
||||
552
@xen-orchestra/qa-test/utils/esxiMigrationUtils.js
Normal file
552
@xen-orchestra/qa-test/utils/esxiMigrationUtils.js
Normal file
@@ -0,0 +1,552 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { once } from 'node:events'
|
||||
import { Readable } from 'node:stream'
|
||||
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { readChunk, readChunkStrict } from '@vates/read-chunk'
|
||||
import Esxi from '@xen-orchestra/vmware-explorer/esxi.mjs'
|
||||
|
||||
import { formatBytes } from './exportUtils.js'
|
||||
import { waitUntil } from './index.js'
|
||||
|
||||
const log = createLogger('xo:qa-test:esxi-migration')
|
||||
|
||||
// =============================================================================
|
||||
// CONFIGURATION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Environment variables without which the ESXi migration tests cannot run at all.
|
||||
* @constant {ReadonlyArray<string>}
|
||||
*/
|
||||
export const REQUIRED_ESXI_ENV = ['ESXI_HOST', 'ESXI_USER', 'ESXI_PASSWORD', 'ESXI_VM_ID']
|
||||
|
||||
/**
|
||||
* Tells whether the ESXi migration tests can run.
|
||||
*
|
||||
* Returned in the shape node:test expects for its `skip` option, so the suite is skipped
|
||||
* with an actionable message rather than failing on an unconfigured environment.
|
||||
*
|
||||
* @returns {string|false} Skip reason, or false when everything needed is set
|
||||
*/
|
||||
export function getEsxiSkipReason() {
|
||||
const missing = REQUIRED_ESXI_ENV.filter(name => (process.env[name] ?? '') === '')
|
||||
return missing.length === 0 ? false : `ESXi migration tests need ${missing.join(', ')} to be set in .env`
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the ESXi source configuration from the environment.
|
||||
*
|
||||
* Only call this once `getEsxiSkipReason()` returned false.
|
||||
*
|
||||
* @returns {{ host: string, user: string, password: string, vmId: string, sslVerify: boolean, timeout: number }}
|
||||
*/
|
||||
export function getEsxiConfig() {
|
||||
return {
|
||||
host: process.env.ESXI_HOST,
|
||||
user: process.env.ESXI_USER,
|
||||
password: process.env.ESXI_PASSWORD,
|
||||
vmId: process.env.ESXI_VM_ID,
|
||||
// lab ESXi hosts serve a self-signed certificate, so this defaults to off
|
||||
sslVerify: process.env.ESXI_SSL_VERIFY === 'true',
|
||||
timeout: millisecondsFromEnv('ESXI_TEST_TIMEOUT_MS', 4 * 3600_000),
|
||||
// how long the guest is given to boot and flush after the reset that churns its disk
|
||||
resetSettleMs: millisecondsFromEnv('ESXI_RESET_SETTLE_MS', 90_000),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an optional environment variable, treating a blank value as unset.
|
||||
*
|
||||
* A key present but left empty in a .env file reads as an empty string, not undefined, so
|
||||
* `??` does not fall back to the default. That matters most for the numeric options:
|
||||
* `Number('')` is 0, which silently turns a timeout into "abort immediately".
|
||||
*
|
||||
* @param {string} name - Environment variable name
|
||||
* @returns {string|undefined} The value, or undefined when unset or empty
|
||||
*/
|
||||
export function optionalEnv(name) {
|
||||
const value = process.env[name]
|
||||
return value === undefined || value === '' ? undefined : value
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an optional duration from the environment, in milliseconds.
|
||||
*
|
||||
* @param {string} name - Environment variable name
|
||||
* @param {number} fallback - Value to use when unset or empty
|
||||
* @returns {number} A positive, finite number of milliseconds
|
||||
* @throws {Error} If the variable is set to something that is not a positive number
|
||||
*/
|
||||
function millisecondsFromEnv(name, fallback) {
|
||||
const raw = optionalEnv(name)
|
||||
if (raw === undefined) {
|
||||
return fallback
|
||||
}
|
||||
|
||||
const value = Number(raw)
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive number of milliseconds, got ${JSON.stringify(raw)}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for a VM's blocked operations to reach an expected state.
|
||||
*
|
||||
* XO's object cache is fed by XAPI events, so a field written during an API call is not
|
||||
* necessarily visible on the very next read — polling here rather than asserting once avoids
|
||||
* failing on that lag, while still failing if the value never changes.
|
||||
*
|
||||
* @param {import('../client/dispatchClient.js').DispatchClient} dispatchClient
|
||||
* @param {string} vmUuid - UUID of the VM
|
||||
* @param {{ blocked: boolean }} expected - Whether `start`/`start_on` must be blocked
|
||||
* @param {number} [timeout=30000] - How long to wait, in milliseconds
|
||||
* @returns {Promise<Object>} The VM record once it matches
|
||||
* @throws {Error} If the expected state is not reached in time
|
||||
*/
|
||||
export async function waitForBlockedOperations(dispatchClient, vmUuid, { blocked }, timeout = 30_000) {
|
||||
let lastSeen
|
||||
try {
|
||||
return await waitUntil(
|
||||
async () => {
|
||||
const vm = await dispatchClient.vm.details(vmUuid)
|
||||
lastSeen = vm.blockedOperations
|
||||
const isBlocked = lastSeen?.start !== undefined && lastSeen?.start_on !== undefined
|
||||
const isUnblocked = lastSeen?.start === undefined && lastSeen?.start_on === undefined
|
||||
return (blocked ? isBlocked : isUnblocked) ? vm : false
|
||||
},
|
||||
1000,
|
||||
timeout
|
||||
)
|
||||
} catch (cause) {
|
||||
const error = new Error(
|
||||
`VM ${vmUuid} should have had start and start_on ${blocked ? 'blocked' : 'unblocked'} within ${timeout} ms, ` +
|
||||
`last seen blockedOperations: ${JSON.stringify(lastSeen)}`
|
||||
)
|
||||
error.cause = cause
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ESXI SOURCE INSPECTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Opens a connection to the source ESXi host.
|
||||
*
|
||||
* Used to assert the state of the *source* VM, which XO's API does not expose in full
|
||||
* (`esxi.listVms` reports the power state but not the snapshots).
|
||||
*
|
||||
* @param {{ host: string, user: string, password: string, sslVerify: boolean }} config
|
||||
* @returns {Promise<Esxi>} A connected client
|
||||
*/
|
||||
export async function connectToEsxi({ host, user, password, sslVerify }) {
|
||||
const esxi = new Esxi(host, user, password, sslVerify)
|
||||
try {
|
||||
await once(esxi, 'ready')
|
||||
} catch (cause) {
|
||||
const error = new Error(
|
||||
`could not connect to ESXi ${host} as ${user}: ${cause.message}. Check ESXI_HOST, ESXI_USER and ` +
|
||||
`ESXI_PASSWORD — in .env a value containing # or a trailing space must be wrapped in double quotes`
|
||||
)
|
||||
error.cause = cause
|
||||
throw error
|
||||
}
|
||||
log.debug('Connected to ESXi', { host })
|
||||
return esxi
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts the source VM is in the state the migration scenario starts from: running, and
|
||||
* without any snapshot so that XO has to take its own.
|
||||
*
|
||||
* @param {Esxi} esxi - Connected ESXi client
|
||||
* @param {string} vmId - ESXi VM id (the numeric MoRef, e.g. '34')
|
||||
* @returns {Promise<Object>} The source VM metadata
|
||||
* @throws {assert.AssertionError} If the VM is halted or already has a snapshot
|
||||
*/
|
||||
export async function assertSourceIsRunningWithoutSnapshot(esxi, vmId) {
|
||||
const metadata = await esxi.getTransferableVmMetadata(vmId)
|
||||
logSourceState(metadata, 'before the first migration')
|
||||
|
||||
assert.notEqual(
|
||||
metadata.powerState,
|
||||
'poweredOff',
|
||||
`source VM ${vmId} (${metadata.name_label}) must be running: the scenario migrates a live VM`
|
||||
)
|
||||
assert.equal(
|
||||
metadata.snapshots?.current,
|
||||
undefined,
|
||||
`source VM ${vmId} (${metadata.name_label}) must have no snapshot so that XO takes its own — ` +
|
||||
`remove its snapshots on the ESXi side before running this test`
|
||||
)
|
||||
|
||||
return metadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Reboots the guest of the source VM so it writes to its active disk, and waits for those
|
||||
* writes to land.
|
||||
*
|
||||
* A hard reset is used rather than a powerOff/powerOn pair on purpose: the latter closes the
|
||||
* vmdk files, which makes VMware rewrite the delta's CID. That would hide the very situation
|
||||
* this suite exists to cover — a delta still carrying its parent's CID.
|
||||
*
|
||||
* How much a boot writes is guest dependent, hence the settle delay rather than a byte
|
||||
* target; nothing here asserts a minimum, the caller reports what was actually transferred.
|
||||
*
|
||||
* @param {Esxi} esxi - Connected ESXi client
|
||||
* @param {string} vmId - ESXi VM id, must be powered on
|
||||
* @param {{ settleMs: number }} options - How long to let the guest boot and flush
|
||||
*/
|
||||
export async function churnSourceByReset(esxi, vmId, { settleMs }) {
|
||||
log.debug('Resetting the source VM to make its guest write', { vmId, settleMs })
|
||||
await esxi.reset(vmId)
|
||||
await new Promise(resolve => setTimeout(resolve, settleMs))
|
||||
log.debug('Source VM had time to boot and flush', { vmId })
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits until the source VM has a current snapshot.
|
||||
*
|
||||
* Used to know when XO has taken its snapshot during a migration: from that point on
|
||||
* everything the guest writes lands in the active disk, which is what the delta pass moves.
|
||||
*
|
||||
* @param {Esxi} esxi - Connected ESXi client
|
||||
* @param {string} vmId - ESXi VM id
|
||||
* @param {number} [timeout=120000] - How long to wait, in milliseconds
|
||||
* @returns {Promise<string>} uid of the current snapshot
|
||||
*/
|
||||
export async function waitForSourceSnapshot(esxi, vmId, timeout = 120_000) {
|
||||
const uid = await waitUntil(
|
||||
async () => {
|
||||
const { snapshots } = await esxi.getTransferableVmMetadata(vmId)
|
||||
return snapshots?.current ?? false
|
||||
},
|
||||
2000,
|
||||
timeout
|
||||
)
|
||||
log.debug('Source VM has been snapshotted by XO', { vmId, snapshotUid: uid })
|
||||
return uid
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings the source VM back to the state the scenarios start from: running, without any
|
||||
* snapshot.
|
||||
*
|
||||
* The snapshots are removed while the VM is halted so the consolidation does not compete with
|
||||
* a live guest. Note `#waitForTaskEnd` gives a task 60 s, so a very large delta to consolidate
|
||||
* would time out.
|
||||
*
|
||||
* @param {Esxi} esxi - Connected ESXi client
|
||||
* @param {string} vmId - ESXi VM id
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function resetSourceState(esxi, vmId) {
|
||||
const { powerState, snapshots } = await esxi.getTransferableVmMetadata(vmId)
|
||||
|
||||
if (snapshots?.current !== undefined) {
|
||||
log.debug('Removing the snapshots left on the source VM', { vmId, current: snapshots.current })
|
||||
await esxi.removeAllSnapshots(vmId)
|
||||
}
|
||||
|
||||
if (powerState === 'poweredOff') {
|
||||
log.debug('Starting the source VM back up', { vmId })
|
||||
await esxi.powerOn(vmId)
|
||||
await waitUntil(async () => (await esxi.getTransferableVmMetadata(vmId)).powerState !== 'poweredOff', 2000, 120_000)
|
||||
}
|
||||
|
||||
log.debug('Source VM is back to its initial state', { vmId })
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the state of the source VM: power state, snapshots, and the identity of every disk
|
||||
* of every chain.
|
||||
*
|
||||
* `uid` is the vmdk CID and `parentId` its parentCID. VMware creates a snapshot delta with
|
||||
* `CID = parentCID` and only rewrites it once the disk is closed, so seeing a delta whose
|
||||
* `uid` equals its `parentId` — and equals its parent's `uid` — is expected right after XO
|
||||
* snapshots the source. It is also the state that used to make the delta look already
|
||||
* imported, so it is worth having in the log whenever a comparison fails later on.
|
||||
*
|
||||
* @param {Object} metadata - Result of `getTransferableVmMetadata`
|
||||
* @param {string} label - When this snapshot of the state was taken
|
||||
*/
|
||||
export function logSourceState(metadata, label) {
|
||||
const describeDisk = (disk, snapshotUid) => ({
|
||||
snapshotUid,
|
||||
node: disk.node,
|
||||
diskPath: disk.diskPath,
|
||||
uid: disk.uid,
|
||||
parentId: disk.parentId,
|
||||
isFull: disk.isFull,
|
||||
vmdkFormat: disk.vmdkFormat,
|
||||
capacity: disk.capacity !== undefined ? formatBytes(disk.capacity) : undefined,
|
||||
})
|
||||
|
||||
const snapshots = metadata.snapshots?.snapshots ?? []
|
||||
|
||||
// the disks are listed flat rather than nested under their snapshot: logSetup.js inspects
|
||||
// at depth 3, and nesting them would print the interesting part as `[Object]`
|
||||
log.debug(`Source VM state ${label}`, {
|
||||
name: metadata.name_label,
|
||||
powerState: metadata.powerState,
|
||||
currentSnapshot: metadata.snapshots?.current,
|
||||
snapshots: snapshots.map(({ uid, parent, displayName, numDisks }) => ({
|
||||
uid,
|
||||
parent,
|
||||
displayName,
|
||||
numDisks,
|
||||
})),
|
||||
snapshotDisks: snapshots.flatMap(snapshot => (snapshot.disks ?? []).map(disk => describeDisk(disk, snapshot.uid))),
|
||||
activeDisks: metadata.disks.map(disk => describeDisk(disk, undefined)),
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches and logs the current state of the source VM.
|
||||
*
|
||||
* @param {Esxi} esxi - Connected ESXi client
|
||||
* @param {string} vmId - ESXi VM id
|
||||
* @param {string} label - When this snapshot of the state was taken
|
||||
* @returns {Promise<Object>} The source VM metadata
|
||||
*/
|
||||
export async function getAndLogSourceState(esxi, vmId, label) {
|
||||
const metadata = await esxi.getTransferableVmMetadata(vmId)
|
||||
logSourceState(metadata, label)
|
||||
return metadata
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TRANSFER ACCOUNTING
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Parses the transfer report XO appends to a VDI's description on every import pass.
|
||||
*
|
||||
* importDiskChain() writes one line per transfer, ending with the disk of the chain it
|
||||
* started from — `base` for a full import, `snapshot` for a delta on top of a previous
|
||||
* one. That makes it the only *persisted* record of what each migration actually moved.
|
||||
*
|
||||
* @param {string} [nameDescription] - `name_description` of the VDI
|
||||
* @returns {Array<{ megabytes: number, seconds: number, from: 'base' | 'snapshot' }>} One entry per transfer, oldest first
|
||||
*/
|
||||
export function parseTransferReports(nameDescription = '') {
|
||||
const reports = []
|
||||
const regex = /([\d.]+) MB in (\d+) s \([^)]*\) from\s+(base|snapshot)/g
|
||||
|
||||
let match
|
||||
while ((match = regex.exec(nameDescription)) !== null) {
|
||||
reports.push({ megabytes: Number(match[1]), seconds: Number(match[2]), from: match[3] })
|
||||
}
|
||||
return reports
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts a VDI was imported by exactly the expected sequence of transfers.
|
||||
*
|
||||
* @param {{ uuid: string, name_description: string }} vdi - VDI record
|
||||
* @param {ReadonlyArray<'base' | 'snapshot'>} expected - Expected origins, oldest transfer first
|
||||
* @throws {assert.AssertionError} If the sequence does not match
|
||||
*/
|
||||
export function assertTransferSequence(vdi, expected) {
|
||||
const reports = parseTransferReports(vdi.name_description)
|
||||
|
||||
log.debug('Transfers reported by the VDI', {
|
||||
vdi: vdi.uuid,
|
||||
expected: [...expected],
|
||||
reports,
|
||||
nameDescription: vdi.name_description,
|
||||
})
|
||||
|
||||
assert.deepEqual(
|
||||
reports.map(({ from }) => from),
|
||||
[...expected],
|
||||
`VDI ${vdi.uuid} should have been imported by ${expected.length} transfer(s) (${expected.join(' then ')}), ` +
|
||||
`its description reads: ${JSON.stringify(vdi.name_description)}`
|
||||
)
|
||||
return reports
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// DISK INTROSPECTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Lists the disks of a VM, ordered by their position on the bus.
|
||||
*
|
||||
* The VDI name is not usable to pair the disks of two separately migrated VMs — it is
|
||||
* derived from the vmdk extent that was read, which differs between a base import and a
|
||||
* whole-chain import — so the bus position is what makes the pairing meaningful.
|
||||
*
|
||||
* @param {import('../client/dispatchClient.js').DispatchClient} dispatchClient
|
||||
* @param {string} vmUuid - UUID of the VM
|
||||
* @returns {Promise<Array<{ uuid: string, name_label: string, name_description: string, size: number, other_config: Record<string, string>, position: number, device: string }>>}
|
||||
*/
|
||||
export async function getOrderedDisks(dispatchClient, vmUuid) {
|
||||
const { restApiClient } = dispatchClient
|
||||
const vm = await dispatchClient.vm.details(vmUuid)
|
||||
|
||||
const vbds = await Promise.all(
|
||||
(vm.$VBDs ?? []).map(vbdId => restApiClient.get(`/rest/v0/vbds/${vbdId}?fields=device,position,is_cd_drive,VDI`))
|
||||
)
|
||||
|
||||
const disks = []
|
||||
for (const vbd of vbds) {
|
||||
if (vbd.is_cd_drive || vbd.VDI === undefined || vbd.VDI === null) {
|
||||
continue
|
||||
}
|
||||
const vdi = await restApiClient.get(
|
||||
`/rest/v0/vdis/${vbd.VDI}?fields=uuid,name_label,name_description,size,other_config`
|
||||
)
|
||||
disks.push({ ...vdi, position: Number(vbd.position), device: vbd.device })
|
||||
}
|
||||
|
||||
disks.sort((a, b) => a.position - b.position)
|
||||
|
||||
log.debug('Disks of the migrated VM', {
|
||||
vm: vmUuid,
|
||||
disks: disks.map(disk => ({
|
||||
device: disk.device,
|
||||
position: disk.position,
|
||||
vdi: disk.uuid,
|
||||
nameLabel: disk.name_label,
|
||||
size: formatBytes(disk.size),
|
||||
// which source disk XO considers this VDI to hold — the reference the next pass
|
||||
// resumes from
|
||||
esxiDiskPath: disk.other_config?.esxi_diskPath,
|
||||
esxiUuid: disk.other_config?.esxi_uuid,
|
||||
})),
|
||||
})
|
||||
|
||||
return disks
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// RAW EXPORT COMPARISON
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Opens the raw (flat) export of a VDI as a Node stream.
|
||||
*
|
||||
* @param {import('../client/restApiClient.js').RestApiClient} restApiClient
|
||||
* @param {string} vdiUuid - UUID of the VDI to export
|
||||
* @param {{ timeout: number }} options
|
||||
* @returns {Promise<Readable>} The response body
|
||||
* @throws {Error} If the export request is rejected
|
||||
*/
|
||||
async function openRawExport(restApiClient, vdiUuid, { timeout }) {
|
||||
const response = await fetch(`${restApiClient.baseUrl}/rest/v0/vdis/${vdiUuid}.raw`, {
|
||||
method: 'GET',
|
||||
headers: restApiClient.headers,
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = new Error(`raw export of VDI ${vdiUuid} failed: HTTP ${response.status} - ${response.statusText}`)
|
||||
error.code = 'RAW_EXPORT_HTTP_ERROR'
|
||||
error.cause = new Error(await response.text())
|
||||
throw error
|
||||
}
|
||||
|
||||
return Readable.fromWeb(response.body)
|
||||
}
|
||||
|
||||
/**
|
||||
* Offset of the first byte that differs between two buffers.
|
||||
* @private
|
||||
*/
|
||||
function firstDifferingByte(a, b) {
|
||||
const length = Math.min(a.length, b.length)
|
||||
for (let i = 0; i < length; i++) {
|
||||
if (a[i] !== b[i]) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that two VDIs export byte for byte identical raw content.
|
||||
*
|
||||
* Both exports are streamed and compared chunk by chunk — nothing is written to disk, so
|
||||
* this works on disks far larger than the runner's storage. The two reads are issued
|
||||
* concurrently so neither HTTP response sits idle while the other is being consumed.
|
||||
*
|
||||
* @param {import('../client/restApiClient.js').RestApiClient} restApiClient
|
||||
* @param {Object} options
|
||||
* @param {string} options.referenceVdi - UUID of the VDI taken as reference
|
||||
* @param {string} options.candidateVdi - UUID of the VDI to check against it
|
||||
* @param {number} options.size - Virtual size of both VDIs, in bytes
|
||||
* @param {number} [options.chunkSize=4194304] - Comparison granularity
|
||||
* @param {number} [options.timeout] - Per-request timeout in milliseconds
|
||||
* @throws {assert.AssertionError} If the contents differ, with the offset of the first difference
|
||||
*/
|
||||
export async function assertRawExportsAreIdentical(
|
||||
restApiClient,
|
||||
{ referenceVdi, candidateVdi, size, chunkSize = 4 * 1024 * 1024, timeout = 4 * 3600_000 }
|
||||
) {
|
||||
log.debug('Comparing raw exports', {
|
||||
referenceVdi,
|
||||
candidateVdi,
|
||||
size: formatBytes(size),
|
||||
chunkSize: formatBytes(chunkSize),
|
||||
timeoutMinutes: Math.round(timeout / 60_000),
|
||||
})
|
||||
|
||||
const [reference, candidate] = await Promise.all([
|
||||
openRawExport(restApiClient, referenceVdi, { timeout }),
|
||||
openRawExport(restApiClient, candidateVdi, { timeout }),
|
||||
])
|
||||
|
||||
const startTime = Date.now()
|
||||
try {
|
||||
let offset = 0
|
||||
let nextProgressLog = 1024 * 1024 * 1024
|
||||
|
||||
while (offset < size) {
|
||||
const length = Math.min(chunkSize, size - offset)
|
||||
|
||||
// readChunkStrict so that a stream ending early fails here, instead of being read as
|
||||
// a difference in content
|
||||
const [referenceChunk, candidateChunk] = await Promise.all([
|
||||
readChunkStrict(reference, length),
|
||||
readChunkStrict(candidate, length),
|
||||
])
|
||||
|
||||
if (!referenceChunk.equals(candidateChunk)) {
|
||||
assert.fail(
|
||||
`raw exports differ at byte ${offset + firstDifferingByte(referenceChunk, candidateChunk)} of ${size}: ` +
|
||||
`reference VDI ${referenceVdi}, candidate VDI ${candidateVdi}`
|
||||
)
|
||||
}
|
||||
|
||||
offset += length
|
||||
if (offset >= nextProgressLog) {
|
||||
const elapsedSeconds = (Date.now() - startTime) / 1000
|
||||
log.debug('Raw exports identical so far', {
|
||||
compared: formatBytes(offset),
|
||||
total: formatBytes(size),
|
||||
percent: Math.round((offset / size) * 100),
|
||||
throughput: `${formatBytes(offset / elapsedSeconds)}/s`,
|
||||
elapsedSeconds: Math.round(elapsedSeconds),
|
||||
})
|
||||
nextProgressLog += 1024 * 1024 * 1024
|
||||
}
|
||||
}
|
||||
|
||||
const [referenceTail, candidateTail] = await Promise.all([readChunk(reference), readChunk(candidate)])
|
||||
assert.equal(referenceTail, null, `reference VDI ${referenceVdi} exported more than its ${size} bytes`)
|
||||
assert.equal(candidateTail, null, `candidate VDI ${candidateVdi} exported more than its ${size} bytes`)
|
||||
|
||||
log.debug('Raw exports are identical', {
|
||||
size: formatBytes(size),
|
||||
durationSeconds: Math.round((Date.now() - startTime) / 1000),
|
||||
})
|
||||
} finally {
|
||||
reference.destroy()
|
||||
candidate.destroy()
|
||||
}
|
||||
}
|
||||
@@ -450,6 +450,43 @@ export default class Esxi extends EventEmitter {
|
||||
return this.#exec('PowerOnVM_Task', { _this: vmId })
|
||||
}
|
||||
|
||||
/**
|
||||
* Hard resets a running VM, making its guest reboot.
|
||||
*
|
||||
* Unlike a powerOff/powerOn pair this keeps the VM powered on as far as the hypervisor is
|
||||
* concerned, so the vmdk files are never closed: a snapshot delta keeps the CID it was
|
||||
* created with. Useful to make a guest write to its active disk without altering the
|
||||
* identity of the disks of the chain.
|
||||
*
|
||||
* @param {string} vmId - id of the VM, must be powered on
|
||||
*/
|
||||
async reset(vmId) {
|
||||
const res = await this.#exec('ResetVM_Task', { _this: vmId })
|
||||
const taskId = res.returnval.$value
|
||||
try {
|
||||
return await this.#waitForTaskEnd(taskId)
|
||||
} catch (error) {
|
||||
warn('Fail to reset VM', { vmId, error })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes every snapshot of a VM, consolidating their content into the base disks.
|
||||
*
|
||||
* @param {string} vmId - id of the VM
|
||||
*/
|
||||
async removeAllSnapshots(vmId) {
|
||||
const res = await this.#exec('RemoveAllSnapshots_Task', { _this: vmId, consolidate: true })
|
||||
const taskId = res.returnval.$value
|
||||
try {
|
||||
return await this.#waitForTaskEnd(taskId)
|
||||
} catch (error) {
|
||||
warn('Fail to remove the snapshots of VM', { vmId, error })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async snapshot(vmId, name, description) {
|
||||
const res = await this.#exec('CreateSnapshotEx_Task', { _this: vmId, name, description, memory: false })
|
||||
const taskId = res.returnval.$value
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "node --test"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
function set(obj, keyPath, val) {
|
||||
const [key, ...other] = keyPath
|
||||
// key like snapshot0->snapshot9 are grouped in an array snapshots[]
|
||||
const match = key.match(/^(.+)([0-9])$/)
|
||||
|
||||
const match = key.match(/^(.+?)([0-9]+)$/)
|
||||
if (match) {
|
||||
// an array
|
||||
let [, label, index] = match
|
||||
index = parseInt(index)
|
||||
// I like my array names in plural form
|
||||
label += 's'
|
||||
if (!obj[label]) {
|
||||
@@ -17,7 +18,7 @@ function set(obj, keyPath, val) {
|
||||
// it contains objects
|
||||
if (!obj[label][index]) {
|
||||
// and this object is not already initialized
|
||||
obj[label][parseInt(index)] = {}
|
||||
obj[label][index] = {}
|
||||
}
|
||||
set(obj[label][index], other, val)
|
||||
} else {
|
||||
|
||||
133
@xen-orchestra/vmware-explorer/parsers/vmsd.test.mjs
Normal file
133
@xen-orchestra/vmware-explorer/parsers/vmsd.test.mjs
Normal file
@@ -0,0 +1,133 @@
|
||||
import { strict as assert } from 'node:assert'
|
||||
import test from 'node:test'
|
||||
|
||||
import parseVmsd from './vmsd.mjs'
|
||||
|
||||
const { describe, it } = test
|
||||
|
||||
// a real .vmsd, as written by ESXi for a VM with 4 snapshots
|
||||
const VMSD_4_SNAPSHOTS = `.encoding = "UTF-8"
|
||||
snapshot.lastUID = "4"
|
||||
snapshot.current = "4"
|
||||
snapshot0.uid = "1"
|
||||
snapshot0.filename = "test flo-Snapshot1.vmsn"
|
||||
snapshot0.displayName = "SNAPSHOT POST INSTALL"
|
||||
snapshot0.description = "blablabla"
|
||||
snapshot0.createTimeHigh = "388745"
|
||||
snapshot0.createTimeLow = "1991180826"
|
||||
snapshot0.numDisks = "1"
|
||||
snapshot0.disk0.fileName = "test flo_0.vmdk"
|
||||
snapshot0.disk0.node = "scsi0:0"
|
||||
snapshot.numSnapshots = "4"
|
||||
snapshot1.uid = "2"
|
||||
snapshot1.filename = "test flo-Snapshot2.vmsn"
|
||||
snapshot1.parent = "1"
|
||||
snapshot1.displayName = "SECOND"
|
||||
snapshot1.description = "small"
|
||||
snapshot1.numDisks = "1"
|
||||
snapshot1.disk0.fileName = "test flo_0-000001.vmdk"
|
||||
snapshot1.disk0.node = "scsi0:0"
|
||||
snapshot2.uid = "3"
|
||||
snapshot2.filename = "test flo-Snapshot3.vmsn"
|
||||
snapshot2.parent = "2"
|
||||
snapshot2.displayName = "third"
|
||||
snapshot2.numDisks = "1"
|
||||
snapshot2.disk0.fileName = "test flo_0-000002.vmdk"
|
||||
snapshot2.disk0.node = "scsi0:0"
|
||||
snapshot3.uid = "4"
|
||||
snapshot3.filename = "test flo-Snapshot4.vmsn"
|
||||
snapshot3.parent = "3"
|
||||
snapshot3.displayName = "from cli"
|
||||
snapshot3.numDisks = "1"
|
||||
snapshot3.disk0.fileName = "test flo_0-000003.vmdk"
|
||||
snapshot3.disk0.node = "scsi0:0"
|
||||
`
|
||||
|
||||
// VMware allows up to 32 snapshots per VM, so the `snapshotN` index is not limited to one digit
|
||||
function buildVmsd(nSnapshots, nDisksPerSnapshot = 1) {
|
||||
const lines = ['.encoding = "UTF-8"', `snapshot.lastUID = "${nSnapshots}"`, `snapshot.current = "${nSnapshots}"`]
|
||||
for (let index = 0; index < nSnapshots; index++) {
|
||||
const uid = index + 1
|
||||
lines.push(
|
||||
`snapshot${index}.uid = "${uid}"`,
|
||||
`snapshot${index}.filename = "vm-Snapshot${uid}.vmsn"`,
|
||||
`snapshot${index}.displayName = "snapshot ${uid}"`,
|
||||
`snapshot${index}.numDisks = "${nDisksPerSnapshot}"`
|
||||
)
|
||||
if (index > 0) {
|
||||
lines.push(`snapshot${index}.parent = "${index}"`)
|
||||
}
|
||||
for (let diskIndex = 0; diskIndex < nDisksPerSnapshot; diskIndex++) {
|
||||
lines.push(
|
||||
`snapshot${index}.disk${diskIndex}.fileName = "vm_${diskIndex}-${String(uid).padStart(6, '0')}.vmdk"`,
|
||||
`snapshot${index}.disk${diskIndex}.node = "scsi0:${diskIndex}"`
|
||||
)
|
||||
}
|
||||
}
|
||||
lines.push(`snapshot.numSnapshots = "${nSnapshots}"`)
|
||||
return lines.join('\n') + '\n'
|
||||
}
|
||||
|
||||
describe('parseVmsd', function () {
|
||||
it('returns undefined when the VM has no current snapshot', function () {
|
||||
assert.equal(parseVmsd('.encoding = "UTF-8"\nsnapshot.lastUID = "0"\nsnapshot.numSnapshots = "0"\n'), undefined)
|
||||
})
|
||||
|
||||
it('parses a VM with less than 10 snapshots', function () {
|
||||
const { current, lastUID, numSnapshots, snapshots } = parseVmsd(VMSD_4_SNAPSHOTS)
|
||||
|
||||
assert.equal(current, '4')
|
||||
assert.equal(lastUID, '4')
|
||||
assert.equal(numSnapshots, '4')
|
||||
assert.deepEqual(
|
||||
snapshots.map(({ uid }) => uid),
|
||||
['1', '2', '3', '4']
|
||||
)
|
||||
assert.deepEqual(snapshots[3], {
|
||||
uid: '4',
|
||||
filename: 'test flo-Snapshot4.vmsn',
|
||||
parent: '3',
|
||||
displayName: 'from cli',
|
||||
numDisks: '1',
|
||||
disks: [{ fileName: 'test flo_0-000003.vmdk', node: 'scsi0:0' }],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses a VM with 10 snapshots or more', function () {
|
||||
const { current, snapshots } = parseVmsd(buildVmsd(12))
|
||||
|
||||
assert.equal(current, '12')
|
||||
assert.equal(snapshots.length, 12)
|
||||
// snapshot10 and snapshot11 must not be mistaken for an index of snapshot1
|
||||
assert.deepEqual(
|
||||
snapshots.map(({ uid }) => uid),
|
||||
['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']
|
||||
)
|
||||
assert.deepEqual(snapshots[10], {
|
||||
uid: '11',
|
||||
filename: 'vm-Snapshot11.vmsn',
|
||||
parent: '10',
|
||||
displayName: 'snapshot 11',
|
||||
numDisks: '1',
|
||||
disks: [{ fileName: 'vm_0-000011.vmdk', node: 'scsi0:0' }],
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps the snapshot referenced by `current` findable by its uid', function () {
|
||||
// buildDiskChainByNode() looks the current snapshot up by uid, and dereferences its disks
|
||||
const { current, snapshots } = parseVmsd(buildVmsd(12))
|
||||
const currentSnapshot = snapshots.find(({ uid }) => uid === current)
|
||||
|
||||
assert.notEqual(currentSnapshot, undefined)
|
||||
assert.equal(currentSnapshot.disks.length, 1)
|
||||
})
|
||||
|
||||
it('parses a snapshot with 10 disks or more', function () {
|
||||
const { snapshots } = parseVmsd(buildVmsd(1, 11))
|
||||
|
||||
assert.equal(snapshots.length, 1)
|
||||
assert.equal(snapshots[0].disks.length, 11)
|
||||
// disk10 must not be mistaken for an index of disk1
|
||||
assert.deepEqual(snapshots[0].disks[10], { fileName: 'vm_10-000001.vmdk', node: 'scsi0:10' })
|
||||
})
|
||||
})
|
||||
@@ -24,6 +24,8 @@
|
||||
- [Immutable backups] Release disks that stayed immutable forever after their metadata was deleted by the retention, or after a merge renamed them, which prevented any further merge or deletion of that disk's backups (PR [#10182](https://github.com/vatesfr/xen-orchestra/pull/10182))
|
||||
- [Plugins/load balancer] No longer try to migrate VMs to disabled host (PR [#10209](https://github.com/vatesfr/xen-orchestra/pull/10209))
|
||||
- [V2V] Improve performance on big VM (>3 To) imports by improving Nbd disk handling (PR [#10157](https://github.com/vatesfr/xen-orchestra/pull/10157))
|
||||
- [Import/VMware] Fix migration of a VM having 10 snapshots or more, or a snapshot with 10 disks or more: the extra snapshots and disks were silently ignored
|
||||
- [Import/VMware] Fix migration of a running VM with "Stop the source VM" enabled and no pre-existing snapshot: the data written since the snapshot taken by XO was not transferred, and the second transfer reported `Nothing to import in this chain`
|
||||
|
||||
### Packages to release
|
||||
|
||||
@@ -46,7 +48,9 @@
|
||||
- @xen-orchestra/backups patch
|
||||
- @xen-orchestra/immutable-backups patch
|
||||
- @xen-orchestra/rest-api minor
|
||||
- @xen-orchestra/vmware-explorer minor
|
||||
- @xen-orchestra/web minor
|
||||
- xo-server patch
|
||||
- xo-server-load-balancer patch
|
||||
|
||||
<!--packages-end-->
|
||||
|
||||
54
packages/xo-server/src/xo-mixins/vmware/_diskIdentity.mjs
Normal file
54
packages/xo-server/src/xo-mixins/vmware/_diskIdentity.mjs
Normal file
@@ -0,0 +1,54 @@
|
||||
// A VDI records which VMware disk of the chain its content was imported from, so that a
|
||||
// later pass can tell where to continue from:
|
||||
// - the delta transfer done after the source VM has been stopped
|
||||
// - a run resuming a migration interrupted by an error
|
||||
//
|
||||
// The vmdk CID must NOT be used for that: VMware creates a snapshot delta with
|
||||
// `CID = parentCID`, and only rewrites it when the disk is closed after having been
|
||||
// written to. Right after XO takes its own snapshot, the base disk and the active delta
|
||||
// therefore advertise the same CID, and the delta wrongly looks already imported: the
|
||||
// data written since the snapshot was silently left on the VMware side.
|
||||
//
|
||||
// The path of the disk is unique inside a chain, and stable for a given snapshot.
|
||||
export const VDI_DISK_PATH_KEY = 'esxi_diskPath'
|
||||
|
||||
// only kept to resume a migration started by an XO that did not store the disk path yet
|
||||
export const VDI_LEGACY_CID_KEY = 'esxi_uuid'
|
||||
|
||||
/**
|
||||
* @param {ReadonlyArray<{ other_config: Record<string, string> }>} existingVdis VDIs already attached to the target VM
|
||||
* @param {Readonly<{ diskPath: string, uid: string }>} vmdkDisk a disk of the VMware chain
|
||||
* @returns {{ other_config: Record<string, string> } | undefined} the VDI holding this disk's content, if any
|
||||
*/
|
||||
export function diskIsAlreadyImported(existingVdis, vmdkDisk) {
|
||||
return existingVdis.find(vdi => {
|
||||
const otherConfig = vdi?.other_config
|
||||
if (otherConfig === undefined) {
|
||||
return false
|
||||
}
|
||||
const importedDiskPath = otherConfig[VDI_DISK_PATH_KEY]
|
||||
if (importedDiskPath !== undefined) {
|
||||
return importedDiskPath === vmdkDisk.diskPath
|
||||
}
|
||||
const { uid } = vmdkDisk
|
||||
if (uid === undefined || uid === vmdkDisk.parentId) {
|
||||
// either the descriptor carries no CID, or this disk is a delta that has not been
|
||||
// closed yet and still carries the CID of its parent: in both cases the CID cannot
|
||||
// identify it. Reporting it as not imported is always the safe answer, it makes the
|
||||
// caller start from a shallower disk of the chain
|
||||
return false
|
||||
}
|
||||
return otherConfig[VDI_LEGACY_CID_KEY] === uid
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Index, in the chain, of the deepest disk whose content has already been imported.
|
||||
*
|
||||
* @param {ReadonlyArray<{ other_config: Record<string, string> }>} existingVdis VDIs already attached to the target VM
|
||||
* @param {ReadonlyArray<{ diskPath: string, uid: string }>} chainByNode base disk first, active disk last
|
||||
* @returns {number} -1 when nothing of this chain has been imported yet
|
||||
*/
|
||||
export function findPreviouslyImportedIndex(existingVdis, chainByNode) {
|
||||
return chainByNode.findLastIndex(disk => diskIsAlreadyImported(existingVdis, disk) !== undefined)
|
||||
}
|
||||
108
packages/xo-server/src/xo-mixins/vmware/_diskIdentity.test.mjs
Normal file
108
packages/xo-server/src/xo-mixins/vmware/_diskIdentity.test.mjs
Normal file
@@ -0,0 +1,108 @@
|
||||
import { strict as assert } from 'node:assert'
|
||||
import test from 'node:test'
|
||||
|
||||
import {
|
||||
diskIsAlreadyImported,
|
||||
findPreviouslyImportedIndex,
|
||||
VDI_DISK_PATH_KEY,
|
||||
VDI_LEGACY_CID_KEY,
|
||||
} from './_diskIdentity.mjs'
|
||||
|
||||
const { describe, it } = test
|
||||
|
||||
// the chain of a running VM, right after XO took its own snapshot: the base disk went
|
||||
// read-only and the delta became the active disk.
|
||||
// both advertise the same CID, as observed on ESXi 7: VMware creates the delta with
|
||||
// `CID = parentCID` and only rewrites it when the disk is closed after being written to
|
||||
const BASE_DISK = {
|
||||
diskPath: 'Linux-AR/Linux-AR.vmdk',
|
||||
uid: 'b4cf9e4e',
|
||||
parentId: 'ffffffff',
|
||||
isFull: true,
|
||||
}
|
||||
const DELTA_DISK = {
|
||||
diskPath: 'Linux-AR/Linux-AR-000001.vmdk',
|
||||
uid: 'b4cf9e4e',
|
||||
parentId: 'b4cf9e4e',
|
||||
isFull: false,
|
||||
}
|
||||
const CHAIN = [BASE_DISK, DELTA_DISK]
|
||||
|
||||
const vdi = other_config => ({ $ref: 'OpaqueRef:vdi', other_config })
|
||||
const importedVdi = disk => vdi({ [VDI_DISK_PATH_KEY]: disk.diskPath, [VDI_LEGACY_CID_KEY]: disk.uid })
|
||||
|
||||
describe('diskIsAlreadyImported', function () {
|
||||
it('matches the disk the VDI was imported from', function () {
|
||||
assert.notEqual(diskIsAlreadyImported([importedVdi(BASE_DISK)], BASE_DISK), undefined)
|
||||
})
|
||||
|
||||
it('does not match another disk of the chain sharing its CID', function () {
|
||||
// the regression: matching on the CID alone made the delta look already imported, and
|
||||
// the data written since the snapshot was silently left on the VMware side
|
||||
assert.equal(diskIsAlreadyImported([importedVdi(BASE_DISK)], DELTA_DISK), undefined)
|
||||
})
|
||||
|
||||
it('returns undefined when no VDI holds this disk', function () {
|
||||
assert.equal(diskIsAlreadyImported([vdi({})], BASE_DISK), undefined)
|
||||
assert.equal(diskIsAlreadyImported([], BASE_DISK), undefined)
|
||||
})
|
||||
|
||||
it('ignores a VDI without other_config', function () {
|
||||
assert.equal(diskIsAlreadyImported([undefined, vdi(undefined)], BASE_DISK), undefined)
|
||||
})
|
||||
|
||||
it('ignores a VDI that XO did not import, even for a disk without a CID', function () {
|
||||
// `undefined === undefined` would report the disk as imported and skip its transfer
|
||||
const foreignVdi = vdi({ name: 'attached by the user' })
|
||||
|
||||
assert.equal(diskIsAlreadyImported([foreignVdi], { diskPath: 'Linux-AR/no-cid.vmdk' }), undefined)
|
||||
})
|
||||
|
||||
it('tells the disks of a multi-disk VM apart', function () {
|
||||
const otherBase = { diskPath: 'Linux-AR/Linux-AR_1.vmdk', uid: 'b4cf9e4e' }
|
||||
const vdis = [importedVdi(BASE_DISK), importedVdi(otherBase)]
|
||||
|
||||
assert.equal(diskIsAlreadyImported(vdis, BASE_DISK).other_config[VDI_DISK_PATH_KEY], BASE_DISK.diskPath)
|
||||
assert.equal(diskIsAlreadyImported(vdis, otherBase).other_config[VDI_DISK_PATH_KEY], otherBase.diskPath)
|
||||
})
|
||||
|
||||
describe('VDI imported by an XO that did not store the disk path', function () {
|
||||
it('falls back to the CID', function () {
|
||||
assert.notEqual(diskIsAlreadyImported([vdi({ [VDI_LEGACY_CID_KEY]: 'b4cf9e4e' })], BASE_DISK), undefined)
|
||||
})
|
||||
|
||||
it('does not fall back to the CID once the disk path is known', function () {
|
||||
const staleCid = vdi({ [VDI_DISK_PATH_KEY]: 'Linux-AR/Linux-AR_1.vmdk', [VDI_LEGACY_CID_KEY]: 'b4cf9e4e' })
|
||||
|
||||
assert.equal(diskIsAlreadyImported([staleCid], BASE_DISK), undefined)
|
||||
})
|
||||
|
||||
it('never matches a delta still carrying the CID of its parent', function () {
|
||||
const legacyVdi = vdi({ [VDI_LEGACY_CID_KEY]: 'b4cf9e4e' })
|
||||
|
||||
assert.equal(diskIsAlreadyImported([legacyVdi], DELTA_DISK), undefined)
|
||||
// so the transfer restarts from the base disk instead of being skipped altogether
|
||||
assert.equal(findPreviouslyImportedIndex([legacyVdi], CHAIN), 0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('findPreviouslyImportedIndex', function () {
|
||||
it('returns -1 when nothing of the chain has been imported', function () {
|
||||
assert.equal(findPreviouslyImportedIndex([], CHAIN), -1)
|
||||
})
|
||||
|
||||
it('points at the base disk after the cold transfer, so the delta is still imported', function () {
|
||||
// with a CID-based match this returned 1 (the length of the chain minus one), which
|
||||
// importDiskChain() reads as 'nothing to import' and returns early
|
||||
assert.equal(findPreviouslyImportedIndex([importedVdi(BASE_DISK)], CHAIN), 0)
|
||||
})
|
||||
|
||||
it('points at the active disk once the whole chain has been imported', function () {
|
||||
assert.equal(findPreviouslyImportedIndex([importedVdi(DELTA_DISK)], CHAIN), CHAIN.length - 1)
|
||||
})
|
||||
|
||||
it('returns the deepest imported disk', function () {
|
||||
assert.equal(findPreviouslyImportedIndex([importedVdi(DELTA_DISK), importedVdi(BASE_DISK)], CHAIN), 1)
|
||||
})
|
||||
})
|
||||
@@ -6,6 +6,12 @@ import { NbdDisk } from '@vates/nbd-client/NbdDisk.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { toQcow2Stream } from '@xen-orchestra/qcow2'
|
||||
import { TaskProgressHandler } from '@xen-orchestra/backups/_runners/_vmRunners/_TaskProgressHandler.mjs'
|
||||
import {
|
||||
diskIsAlreadyImported,
|
||||
findPreviouslyImportedIndex,
|
||||
VDI_DISK_PATH_KEY,
|
||||
VDI_LEGACY_CID_KEY,
|
||||
} from './_diskIdentity.mjs'
|
||||
|
||||
const { warn } = createLogger('xo:importdiskfromdatastore')
|
||||
|
||||
@@ -77,10 +83,10 @@ async function importDiskChain({ esxi, sr, vm, chainByNode, userdevice, vmId })
|
||||
Task.info(`Importing disk in ${format} format, with block of ${blockSize} bytes`)
|
||||
|
||||
let dataMap
|
||||
const previouslyImportedIndex = chainByNode.findLastIndex(disk => !!diskIsAlreadyImported(existingVdis, disk))
|
||||
const previouslyImportedIndex = findPreviouslyImportedIndex(existingVdis, chainByNode)
|
||||
let existingVdi
|
||||
if (previouslyImportedIndex === chainByNode.length - 1) {
|
||||
Task.info('Nothing to import in this chain')
|
||||
Task.info(`Nothing to import in this chain, ${diskPath} has already been imported`)
|
||||
return
|
||||
}
|
||||
if (previouslyImportedIndex !== -1) {
|
||||
@@ -147,18 +153,16 @@ async function importDiskChain({ esxi, sr, vm, chainByNode, userdevice, vmId })
|
||||
`${existingVdi.name_description}
|
||||
${transfered} MB in ${duration} s (${speed}MB/s) from ${previouslyImportedIndex === -1 ? 'base' : 'snapshot'}`
|
||||
)
|
||||
await sr.$xapi.setFieldEntries('VDI', existingVdi.$ref, 'other_config', { esxi_uuid: uid })
|
||||
await sr.$xapi.setFieldEntries('VDI', existingVdi.$ref, 'other_config', {
|
||||
[VDI_DISK_PATH_KEY]: diskPath,
|
||||
[VDI_LEGACY_CID_KEY]: uid,
|
||||
})
|
||||
} catch (err) {
|
||||
Task.warning(err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
function diskIsAlreadyImported(vdis, vmdkDisk) {
|
||||
// look for a vdi with the right contentId
|
||||
return vdis.find(vdi => vdi?.other_config.esxi_uuid === vmdkDisk.uid)
|
||||
}
|
||||
|
||||
export const importDisksFromDatastore = async function importDisksFromDatastore({ esxi, vm, vmId, chainsByNodes, sr }) {
|
||||
return await Promise.all(
|
||||
Object.keys(chainsByNodes).map(async (node, userdevice) =>
|
||||
|
||||
@@ -178,7 +178,7 @@ export default class MigrateVm {
|
||||
await vm.$snapshot({ name_label: `after ${isRunning ? 'partial' : 'complete'} import from V2V` })
|
||||
if (isRunning) {
|
||||
if (stopSource) {
|
||||
await esxi.powerOff(vmId)
|
||||
await Task.run({ properties: { name: 'stopping source VM' } }, async () => esxi.powerOff(vmId))
|
||||
await importDisksFromDatastore({
|
||||
chainsByNodes,
|
||||
esxi,
|
||||
@@ -188,6 +188,8 @@ export default class MigrateVm {
|
||||
})
|
||||
await sr.$xapi.setFieldEntries('VM', vm.$ref, 'other_config', { sourceVmId: vmId, sourceSnapshotId: null })
|
||||
await vm.$snapshot({ name_label: 'complete import from V2V' })
|
||||
} else {
|
||||
await Task.info(`VM was running, no stop source. We'll keep the partial import.`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user