feat(v2v): enable CBT automatically + use soft shutdown when possible (#10637)

This commit is contained in:
Florent BEAUCHAMP
2026-10-27 10:24:16 +01:00
committed by GitHub
parent 0be905e7cf
commit c2bda04112
5 changed files with 454 additions and 5 deletions

View File

@@ -908,6 +908,85 @@ export default class Esxi extends EventEmitter {
return this.#runTask('PowerOffVM_Task', { _this: vmId }, { signal, timeout })
}
/**
* Shuts the guest down cleanly through the VMware tools, and waits for the VM to be off.
*
* A power off is the guest losing power: Windows leaves its NTFS volumes dirty, which then cannot be
* written offline, e.g. by the repair ISO. The VM is powered off anyway when its tools do not run, when
* the guest refuses to shut down, or does not within `timeout`.
*
* @param {string} vmId - id of the VM, powered on
* @param {object} [options]
* @param {number} [options.timeout] - in ms, how long the guest is given to shut down
* @param {number} [options.pollInterval] - in ms, between two reads of the power state
* @param {AbortSignal} [options.signal]
* @returns {Promise<'shutdown' | 'powerOff'>} how the VM was stopped
*/
async shutdown(vmId, { signal, timeout = 10 * 60e3, pollInterval = 5e3 } = {}) {
const toolsStatus = await this.#retrieveProperty('VirtualMachine', vmId, 'guest.toolsRunningStatus', {
signal,
}).catch(error => {
signal?.throwIfAborted()
warn('failed to read the state of the VMware tools', { error, vmId })
})
if (toolsStatus === 'guestToolsRunning') {
try {
// not a task: the call returns once the guest has been asked to shut down
await this.#vimClient.call('ShutdownGuest', { _this: vmId })
await this.#waitPoweredOff(vmId, { signal, timeout, pollInterval })
return 'shutdown'
} catch (error) {
signal?.throwIfAborted()
warn('the guest did not shut down, powering it off', { error, vmId })
}
} else {
info('the VMware tools do not run, powering the VM off', { toolsStatus, vmId })
}
try {
await this.powerOff(vmId, { signal })
} catch (error) {
signal?.throwIfAborted()
// the host refuses to power off a VM which is already off: the guest may have completed a shut
// down whose request failed on our side, or the VM may have been stopped since the caller read
// its power state, possibly long before
const powerState = await this.#retrieveProperty('VirtualMachine', vmId, 'runtime.powerState', { signal })
if (powerState === 'poweredOff') {
return 'shutdown'
}
throw error
}
return 'powerOff'
}
async #waitPoweredOff(vmId, { signal, timeout, pollInterval }) {
const deadline = Date.now() + timeout
for (;;) {
let powerState
try {
powerState = await this.#retrieveProperty('VirtualMachine', vmId, 'runtime.powerState', { signal })
} catch (error) {
// the guest keeps shutting down on the host: a transient failure of a single poll must not
// end in powering off a guest which is about to be off
if (!isRetryableError(error) || Date.now() >= deadline) {
throw error
}
warn('failed to read the power state of a VM, will poll again', { error, vmId })
}
if (powerState === 'poweredOff') {
return
}
if (Date.now() >= deadline) {
const error = new Error(`the VM ${vmId} is still running ${timeout}ms after its guest was asked to shut down`)
error.code = 'SHUTDOWN_TIMEOUT'
error.vmId = vmId
throw error
}
await delay(pollInterval, undefined, { signal })
}
}
/**
* Powers on a VM, and waits for the hypervisor to have started it.
*
@@ -972,6 +1051,58 @@ export default class Esxi extends EventEmitter {
)
}
/**
* Removes a single snapshot, its children are kept, and consolidates its disks.
*
* @param {string} snapshotId - id of the snapshot, e.g. the `result` of {@link snapshot}
* @param {object} [options]
* @param {number} [options.timeout] - in ms
* @param {AbortSignal} [options.signal]
* @returns {Promise<object>} the `info` of the successful task
*/
async removeSnapshot(snapshotId, { signal, timeout = 30 * 60e3 } = {}) {
return this.#runTask(
'RemoveSnapshot_Task',
{ _this: snapshotId, removeChildren: false, consolidate: true },
{ signal, timeout }
)
}
/**
* Turns the change tracking (CBT) of a VM on, when it is not already.
*
* The host only starts tracking when the disks are next opened, e.g. by a snapshot, running or
* not: until then it has no tracking file to answer from, and a snapshot taken before carries no
* `changeId`, no delta can be computed from it.
*
* @param {string} vmId - id of the VM
* @param {object} [options]
* @param {number} [options.timeout] - in ms
* @param {AbortSignal} [options.signal]
* @returns {Promise<boolean>} whether this call turned the tracking on
*/
async enableChangeTracking(vmId, { signal, timeout = 5 * 60e3 } = {}) {
const enabled = await this.#retrieveProperty('VirtualMachine', vmId, 'config.changeTrackingEnabled', {
signal,
}).catch(error => {
// absent when the VM has never been tracked
if (error.code !== 'NO_PROPERTY') {
throw error
}
})
if (isTrue(enabled)) {
return false
}
await this.#runTask(
'ReconfigVM_Task',
{ _this: vmId, spec: virtualMachineConfigSpec({ changeTrackingEnabled: true }) },
{ signal, timeout }
)
info('change tracking enabled', { vmId })
return true
}
/**
* Attaches an existing vmdk to a VM, powered on or not, without ever writing to it.
*
@@ -1495,6 +1626,49 @@ export default class Esxi extends EventEmitter {
}
}
/**
* Whether the change tracking of a VM already answers for every disk a transfer would read, in
* which case no snapshot is needed to start it.
*
* - powered on: the disks are read up to the current snapshot, which must have recorded a
* `changeId` for each of them, the precondition of the delta which follows
* - powered off: the host is asked for every used block of each disk. It answers the same
* FileFault whether the tracking was never started, or reset: a snapshot cures both
*
* An independent disk is left out: it is never snapshotted, so a snapshot cannot start its tracking.
*
* @param {string} vmId - id of the VM, its change tracking enabled
* @param {object} [options]
* @param {AbortSignal} [options.signal]
* @returns {Promise<boolean>}
*/
async isChangeTrackingReady(vmId, { signal } = {}) {
const { currentSnapshotId, devices, powerState, snapshotDevices } = await this.getChangeTracking(vmId, { signal })
const isSnapshotted = device => isVirtualDisk(device) && !device.backing?.diskMode?.startsWith('independent')
if (powerState !== 'poweredOff') {
if (currentSnapshotId === undefined) {
// a powered on VM can only be asked through a snapshot
return false
}
const snapshotDisks = asArray(snapshotDevices.get(currentSnapshotId)?.VirtualDevice).filter(isSnapshotted)
return snapshotDisks.every(disk => disk.backing?.changeId !== undefined)
}
for (const disk of asArray(devices?.VirtualDevice).filter(isSnapshotted)) {
signal?.throwIfAborted()
try {
// the first answer is enough: a disk which is not tracked fails at once, no need to walk it
await this.#queryChangedDiskAreas(vmId, { changeId: '*', deviceKey: Number(disk.key), startOffset: 0 })
} catch (error) {
signal?.throwIfAborted()
info('the change tracking does not answer for a disk', { deviceKey: disk.key, error, vmId })
return false
}
}
return true
}
/**
* Blocks of a disk to read, from the change tracking of the host.
*

View File

@@ -348,6 +348,172 @@ const taskInfo = (state, extra = {}) =>
const startedTask = () => ({ returnval: moRef('Task', 'task-1') })
describe('shutdown', function () {
/**
* @param {object} params
* @param {string} [params.toolsStatus] - `guest.toolsRunningStatus`
* @param {Array<string | Error>} params.powerStates - successive answers for `runtime.powerState`, an
* error is thrown, the last one repeats
* @param {() => object} [params.shutdownGuest] - answer to `ShutdownGuest`, may throw
* @param {string} [params.powerOffState] - state of the `PowerOffVM_Task`
*/
const shutdownEsxi = ({
toolsStatus = 'guestToolsRunning',
powerStates,
shutdownGuest = () => ({}),
powerOffState = 'success',
}) => {
const states = [...powerStates]
return connectedEsxi({
responses: {
ShutdownGuest: shutdownGuest,
PowerOffVM_Task: () => startedTask(),
RetrievePropertiesEx: ({ specSet }) => {
const { pathSet, type } = specSet[0].propSet[0]
if (type === 'Task') {
return powerOffState === 'success'
? taskInfo('success')
: taskInfo(powerOffState, {
error: {
attributes: { 'xsi:type': 'LocalizedMethodFault' },
fault: { attributes: { 'xsi:type': 'InvalidPowerState' } },
localizedMessage: 'The attempted operation cannot be performed in the current state (Powered off).',
},
})
}
if (pathSet[0] === 'guest.toolsRunningStatus') {
return propertyOf('VirtualMachine', 'vm-1', pathSet[0], toolsStatus)
}
const state = states.length > 1 ? states.shift() : states[0]
if (state instanceof Error) {
throw state
}
return propertyOf('VirtualMachine', 'vm-1', pathSet[0], state)
},
},
})
}
it('asks the guest to shut down and waits for it', async function () {
const { esxi, vimClient } = await shutdownEsxi({ powerStates: ['poweredOn', 'poweredOn', 'poweredOff'] })
assert.equal(await esxi.shutdown('vm-1', { pollInterval: 1 }), 'shutdown')
assert.deepEqual(vimClient.callsTo('ShutdownGuest')[0].args, { _this: 'vm-1' })
assert.equal(vimClient.callsTo('PowerOffVM_Task').length, 0)
})
it('powers the VM off when the VMware tools do not run', async function () {
const { esxi, vimClient } = await shutdownEsxi({ toolsStatus: 'guestToolsNotRunning', powerStates: ['poweredOn'] })
assert.equal(await esxi.shutdown('vm-1', { pollInterval: 1 }), 'powerOff')
assert.equal(vimClient.callsTo('ShutdownGuest').length, 0)
assert.equal(vimClient.callsTo('PowerOffVM_Task').length, 1)
})
it('powers the VM off when the guest refuses, or does not shut down in time', async function () {
const refusing = await shutdownEsxi({
powerStates: ['poweredOn'],
shutdownGuest: () => {
throw Object.assign(new Error('ToolsUnavailable'), { code: 'ToolsUnavailable' })
},
})
assert.equal(await refusing.esxi.shutdown('vm-1', { pollInterval: 1 }), 'powerOff')
const stuck = await shutdownEsxi({ powerStates: ['poweredOn'] })
assert.equal(await stuck.esxi.shutdown('vm-1', { pollInterval: 1, timeout: 5 }), 'powerOff')
assert.equal(stuck.vimClient.callsTo('PowerOffVM_Task').length, 1)
})
it('polls again when reading the power state fails transiently', async function () {
const reset = Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })
const { esxi, vimClient } = await shutdownEsxi({ powerStates: ['poweredOn', reset, 'poweredOn', 'poweredOff'] })
assert.equal(await esxi.shutdown('vm-1', { pollInterval: 1 }), 'shutdown')
assert.equal(vimClient.callsTo('PowerOffVM_Task').length, 0)
})
it('succeeds when the power off is refused because the VM is already off', async function () {
// the read while waiting, then the one after the refused power off
const { esxi, vimClient } = await shutdownEsxi({ powerStates: ['poweredOn', 'poweredOff'], powerOffState: 'error' })
assert.equal(await esxi.shutdown('vm-1', { pollInterval: 1, timeout: 0 }), 'shutdown')
assert.equal(vimClient.callsTo('PowerOffVM_Task').length, 1)
})
it('fails when the power off is refused and the VM is still running', async function () {
const { esxi } = await shutdownEsxi({
toolsStatus: 'guestToolsNotRunning',
powerStates: ['poweredOn'],
powerOffState: 'error',
})
await assert.rejects(esxi.shutdown('vm-1', { pollInterval: 1 }))
})
})
describe('removeSnapshot', function () {
it('removes only the snapshot, and consolidates its disks', async function () {
const { esxi, vimClient } = await connectedEsxi({
responses: {
RemoveSnapshot_Task: () => startedTask(),
RetrievePropertiesEx: () => taskInfo('success'),
},
})
await esxi.removeSnapshot('snapshot-1')
assert.deepEqual(vimClient.callsTo('RemoveSnapshot_Task')[0].args, {
_this: 'snapshot-1',
removeChildren: false,
consolidate: true,
})
})
})
describe('enableChangeTracking', function () {
// `undefined` answers the property as absent, like a VM which has never been tracked
const trackingEsxi = enabled =>
connectedEsxi({
responses: {
ReconfigVM_Task: () => startedTask(),
RetrievePropertiesEx: ({ specSet }) => {
const { pathSet, type } = specSet[0].propSet[0]
if (type === 'Task') {
return taskInfo('success')
}
return enabled === undefined
? { returnval: { objects: [{ obj: moRef('VirtualMachine', 'vm-1'), propSet: [] }] } }
: propertyOf('VirtualMachine', 'vm-1', pathSet[0], enabled)
},
},
})
it('turns the tracking on when it is off', async function () {
const { esxi, vimClient } = await trackingEsxi('false')
assert.equal(await esxi.enableChangeTracking('vm-1'), true)
const [{ args }] = vimClient.callsTo('ReconfigVM_Task')
assert.equal(args._this, 'vm-1')
assert.deepEqual(args.spec, {
attributes: { 'xsi:type': 'VirtualMachineConfigSpec' },
changeTrackingEnabled: true,
})
})
it('turns the tracking on when the VM has never been tracked', async function () {
const { esxi, vimClient } = await trackingEsxi(undefined)
assert.equal(await esxi.enableChangeTracking('vm-1'), true)
assert.equal(vimClient.callsTo('ReconfigVM_Task').length, 1)
})
it('does not reconfigure a VM already tracked', async function () {
const { esxi, vimClient } = await trackingEsxi('true')
assert.equal(await esxi.enableChangeTracking('vm-1'), false)
assert.equal(vimClient.callsTo('ReconfigVM_Task').length, 0)
})
})
describe('tasks', function () {
it('resolves with the info of the successful task', async function () {
const { esxi, vimClient } = await connectedEsxi({
@@ -1537,6 +1703,42 @@ RW 16384 VMFSSPARSE "vm-000001-delta.vmdk"
)
assert.equal(vimClient.callsTo('QueryChangedDiskAreas').length, 1)
})
describe('isChangeTrackingReady', function () {
it('is ready when the host answers for every disk of a powered off VM', async function () {
const { esxi, vimClient } = await cbtEsxi()
assert.equal(await esxi.isChangeTrackingReady('vm-1'), true)
// a single answer per disk is enough, the disk is not walked
const queries = vimClient.callsTo('QueryChangedDiskAreas')
assert.equal(queries.length, 1)
assert.deepEqual(queries[0].args, { _this: 'vm-1', deviceKey: 2000, startOffset: 0, changeId: '*' })
})
it('is not ready when the host refuses to answer for a disk of a powered off VM', async function () {
const { esxi } = await cbtEsxi({
queryChangedDiskAreas: () => {
throw Object.assign(new Error('FileFault'), { code: 'FileFault' })
},
})
assert.equal(await esxi.isChangeTrackingReady('vm-1'), false)
})
it('is ready when the current snapshot of a running VM recorded a changeId', async function () {
const { esxi, vimClient } = await cbtEsxi({ powerState: 'poweredOn' })
assert.equal(await esxi.isChangeTrackingReady('vm-1'), true)
assert.equal(vimClient.callsTo('QueryChangedDiskAreas').length, 0)
})
it('is not ready when the current snapshot of a running VM predates the tracking', async function () {
// e.g. a previous run turned the tracking on, then failed before taking its snapshot
const { esxi } = await cbtEsxi({ powerState: 'poweredOn', changeId: null })
assert.equal(await esxi.isChangeTrackingReady('vm-1'), false)
})
})
})
})

View File

@@ -26,7 +26,7 @@ export const QUERY_CHANGED_DISK_AREAS = Object.freeze(['_this', 'snapshot', 'dev
// only the part of the sequence this package sends: the whole type counts dozens of elements, an
// element added here must be inserted at its place in the schema
export const VIRTUAL_MACHINE_CONFIG_SPEC = Object.freeze(['deviceChange'])
export const VIRTUAL_MACHINE_CONFIG_SPEC = Object.freeze(['deviceChange', 'changeTrackingEnabled'])
export const VIRTUAL_DEVICE_CONFIG_SPEC = Object.freeze([
'operation',

View File

@@ -11,12 +11,15 @@
> Users must be able to say: "Nice enhancement, I'm eager to test it"
- [V2V] With `stopSource`, the source VM is shut down cleanly through the VMware tools, and only powered off when its tools do not run or it does not shut down within 10 minutes (PR [#10637](https://github.com/vatesfr/xen-orchestra/pull/10637))
### Bug fixes
> Users must be able to say: "I had this issue, happy to know it's fixed"
- Fix `cannot read properties of undefined (reading code)` (PR [#10668](https://github.com/vatesfr/xen-orchestra/pull/10668))
- [SDN Controller] A Global Private Network that fails to be set up, or whose preferred star-center host was removed, no longer blocks the other private networks (PR [#10629](https://github.com/vatesfr/xen-orchestra/pull/10629))
- [V2V] Fix the migration of a running VM whose change tracking (CBT) is not enabled failing on ESXi 7 and 8 once the source is stopped: the change tracking is now enabled on the source VM, so that only the used blocks are read, and only the changed ones for the delta. A partial import made before can't be resumed and is transferred again (PR [#10637](https://github.com/vatesfr/xen-orchestra/pull/10637))
### Packages to release
@@ -34,6 +37,7 @@
<!--packages-start-->
- @xen-orchestra/vmware-explorer minor
- xo-server patch
- xo-server-sdn-controller patch

View File

@@ -200,7 +200,13 @@ export default class MigrateVm {
await vm.$snapshot({ name_label: `after ${isRunning ? 'partial' : 'complete'} import from V2V` })
if (isRunning) {
if (stopSource) {
await Task.run({ properties: { name: 'stopping source VM' } }, async () => esxi.powerOff(vmId))
await Task.run({ properties: { name: 'stopping source VM' } }, async () => {
// a clean shut down when the guest allows it: a power off leaves the file systems dirty
const stoppedBy = await esxi.shutdown(vmId, { signal: Task.abortSignal })
if (stoppedBy === 'powerOff') {
Task.warning('the guest could not be shut down cleanly, it was powered off')
}
})
await importDisksFromDatastore({
chainsByNodes,
esxi,
@@ -232,12 +238,75 @@ export default class MigrateVm {
return esxi.getTransferableVmMetadata(vmId)
})
if (!metadata.snapshots || !metadata.snapshots.current) {
if (metadata.powerState !== 'poweredOff') {
await Task.run({ properties: { name: `taking a snapshot of ${vmId}` } }, () => {
// the change tracking of the host gives the blocks to read: every block the disk uses for the
// full pass, powered off or not, and what changed since the snapshot for the delta pass. Without
// it, a full pass reads the unallocated blocks too
let isChangeTrackingEnabled = false
let isChangeTrackingNew = false
try {
isChangeTrackingNew = await Task.run({ properties: { name: `enabling change tracking on ${vmId}` } }, () =>
esxi.enableChangeTracking(vmId)
)
isChangeTrackingEnabled = true
} catch (error) {
// not fatal: the disks are read to find their blocks
warn('failed to enable the change tracking on the source', { error, vmId })
Task.warning(`the change tracking could not be enabled on the source: ${error.message}`)
}
const isRunning = metadata.powerState !== 'poweredOff'
// a running VM is only read up to a snapshot
const isSnapshotRequired = isRunning && !metadata.snapshots?.current
// tracking only starts with the next snapshot: before it the host has no tracking file to answer
// from, and an existing snapshot carries no changeId. This is checked on the source rather than
// deduced from this run turning the tracking on: a previous run may have turned it on, then failed
// before its snapshot. A partial import made from a snapshot without changeId is not resumed on
// purpose: its delta can only be read from a COWD extent, which ESXi 7 and later no longer write,
// the transfer starts over
let isSnapshotUseful = isChangeTrackingNew
if (isChangeTrackingEnabled && !isChangeTrackingNew && !isSnapshotRequired) {
isSnapshotUseful = await esxi.isChangeTrackingReady(vmId, { signal: Task.abortSignal }).then(
isReady => !isReady,
error => {
Task.abortSignal?.throwIfAborted()
// not fatal: a snapshot which turns out useless costs less than a transfer without tracking
warn('failed to check the change tracking of the source', { error, vmId })
return true
}
)
}
if (isSnapshotRequired || isSnapshotUseful) {
let snapshotTaskInfo
try {
snapshotTaskInfo = await Task.run({ properties: { name: `taking a snapshot of ${vmId}` } }, () => {
return esxi.snapshot(vmId, '[V2V] migration to XCP-ng', 'created automatically during migration')
})
} catch (error) {
if (isSnapshotRequired) {
throw error
}
Task.abortSignal?.throwIfAborted()
// not fatal, like the change tracking it would have started
warn('failed to snapshot the source to start its change tracking', { error, vmId })
Task.warning(`the source could not be snapshotted to start its change tracking: ${error.message}`)
}
const snapshotId = snapshotTaskInfo?.result?.$value
if (!isRunning && snapshotId !== undefined) {
// only taken to start the tracking, which outlives it: a powered off VM is read as it is,
// nothing would ever remove this snapshot from the source
await Task.run({ properties: { name: `removing the snapshot of ${vmId}` } }, () =>
esxi.removeSnapshot(snapshotId)
).catch(error => {
Task.abortSignal?.throwIfAborted()
// not fatal: the import reads the chain of the disks, this snapshot included
warn('failed to remove the snapshot which started the change tracking', { error, snapshotId, vmId })
Task.warning(`the snapshot ${snapshotId} could not be removed from the source: ${error.message}`)
})
}
if (snapshotTaskInfo !== undefined) {
metadata = await Task.run(
{ properties: { name: `updating metadata of ${vmId} after taking a snapshot` } },
() => esxi.getTransferableVmMetadata(vmId)