diff --git a/@vates/types/src/xo-app.mts b/@vates/types/src/xo-app.mts index f58178d2be..71edd75fe8 100644 --- a/@vates/types/src/xo-app.mts +++ b/@vates/types/src/xo-app.mts @@ -340,8 +340,11 @@ export type XoApp = { readOnly?: XoServer['readOnly'] } ): Promise - rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise - rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise + rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask; shutdownPinnedVms?: boolean }): Promise + rollingPoolUpdate( + pool: XoPool, + opts?: { rebootVm?: boolean; parentTask?: VatesTask; shutdownPinnedVms?: boolean } + ): Promise setVmResourceSet(vmId: XoVm['id'], resourceSetId: string | null, force?: boolean): Promise shareVmResourceSet(vmId: XoVm['id']): Promise removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise diff --git a/@xen-orchestra/rest-api/src/pools/pool.controller.mts b/@xen-orchestra/rest-api/src/pools/pool.controller.mts index 8c5cb2d257..3fd107168f 100644 --- a/@xen-orchestra/rest-api/src/pools/pool.controller.mts +++ b/@xen-orchestra/rest-api/src/pools/pool.controller.mts @@ -31,6 +31,7 @@ import { createdResp, featureUnauthorized, forbiddenOperationResp, + incorrectStateResp, internalServerErrorResp, invalidParameters as invalidParametersResp, noContentResp, @@ -359,22 +360,33 @@ export class PoolController extends XapiXoController { * Required privilege: * - resource: pool, action: rolling-reboot * + * Set `shutdownPinnedVms` to `true` to shut down VMs that cannot be migrated (PCI passthrough, vGPU, SR-IOV VIF) + * before their host reboots and start them again on it afterwards. Without it, such VMs make the action fail + * with an `incorrect state` error listing their UUIDs. + * * @example id "355ee47d-ff4c-4924-3db2-fd86ae629677" + * @example body { "shutdownPinnedVms": true } */ @Example(taskLocation) @Extension('x-mcp-exposure', 'confirm') @Post('{id}/actions/rolling_reboot') - @Middlewares(acl({ resource: 'pool', action: 'rolling-reboot', objectId: 'params.id' })) + @Middlewares([json(), acl({ resource: 'pool', action: 'rolling-reboot', objectId: 'params.id' })]) @SuccessResponse(asynchronousActionResp.status, asynchronousActionResp.description) @Response(forbiddenOperationResp.status, forbiddenOperationResp.description) @Response(noContentResp.status, noContentResp.description) @Response(featureUnauthorized.status, featureUnauthorized.description) @Response(notFoundResp.status, notFoundResp.description) - rollingReboot(@Path() id: string, @Query() sync?: boolean): CreateActionReturnType { + @Response(incorrectStateResp.status, incorrectStateResp.description) + rollingReboot( + @Path() id: string, + @Body() body?: { shutdownPinnedVms?: boolean }, + @Query() sync?: boolean + ): CreateActionReturnType { const poolId = id as XoPool['id'] + const shutdownPinnedVms = body?.shutdownPinnedVms ?? false const action = async (task: VatesTask) => { const pool = this.getObject(poolId) - await this.restApi.xoApp.rollingPoolReboot(pool, { parentTask: task }) + await this.restApi.xoApp.rollingPoolReboot(pool, { parentTask: task, shutdownPinnedVms }) } return this.createAction(action, { @@ -392,22 +404,33 @@ export class PoolController extends XapiXoController { * Required privilege: * - resource: pool, action: rolling-update * + * Set `shutdownPinnedVms` to `true` to shut down VMs that cannot be migrated (PCI passthrough, vGPU, SR-IOV VIF) + * before their host reboots and start them again on it afterwards. Without it, such VMs make the action fail + * with an `incorrect state` error listing their UUIDs. + * * @example id "355ee47d-ff4c-4924-3db2-fd86ae629677" + * @example body { "shutdownPinnedVms": true } */ @Example(taskLocation) @Extension('x-mcp-exposure', 'confirm') @Post('{id}/actions/rolling_update') - @Middlewares(acl({ resource: 'pool', action: 'rolling-update', objectId: 'params.id' })) + @Middlewares([json(), acl({ resource: 'pool', action: 'rolling-update', objectId: 'params.id' })]) @SuccessResponse(asynchronousActionResp.status, asynchronousActionResp.description) @Response(forbiddenOperationResp.status, forbiddenOperationResp.description) @Response(noContentResp.status, noContentResp.description) @Response(featureUnauthorized.status, featureUnauthorized.description) @Response(notFoundResp.status, notFoundResp.description) - rollingUpdate(@Path() id: string, @Query() sync?: boolean): CreateActionReturnType { + @Response(incorrectStateResp.status, incorrectStateResp.description) + rollingUpdate( + @Path() id: string, + @Body() body?: { shutdownPinnedVms?: boolean }, + @Query() sync?: boolean + ): CreateActionReturnType { const poolId = id as XoPool['id'] + const shutdownPinnedVms = body?.shutdownPinnedVms ?? false const action = async (task: VatesTask) => { const pool = this.getObject(poolId) - await this.restApi.xoApp.rollingPoolUpdate(pool, { parentTask: task }) + await this.restApi.xoApp.rollingPoolUpdate(pool, { parentTask: task, shutdownPinnedVms }) } return this.createAction(action, { diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 14d109db7b..cc4a1e7b57 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -12,6 +12,8 @@ > Users must be able to say: "Nice enhancement, I'm eager to test it" - [XO6/Treeview] Fix hosts alignment in Treeview when hosts have different numbers of VMs (PR [#10153](https://github.com/vatesfr/xen-orchestra/pull/10153)) +- [REST API] Possibility of sending `shutdownPinnedVms` in the body of the `/pools/:id/actions/rolling_update` and `rolling_reboot` endpoints (PR [#10125](https://github.com/vatesfr/xen-orchestra/pull/10125)) +- [Rolling Pool Update/Reboot] New `shutdownPinnedVms` option: VMs that cannot be migrated because they use a host-bound device (PCI passthrough, vGPU, SR-IOV VIF) are cleanly shut down before their host reboots and started again on it afterwards, instead of aborting the whole run. When such VMs block the run, XO now lists them and asks for confirmation instead of failing with a raw `CANNOT_EVACUATE_HOST` error (PR [#10125](https://github.com/vatesfr/xen-orchestra/pull/10125)) ### Bug fixes @@ -41,10 +43,14 @@ +- @vates/types minor - @xen-orchestra/backup-archive patch - @xen-orchestra/backups patch - @xen-orchestra/disk-transform patch +- @xen-orchestra/rest-api minor - @xen-orchestra/web minor - @xen-orchestra/web-core minor +- xo-server minor +- xo-web minor diff --git a/packages/xo-server/config.toml b/packages/xo-server/config.toml index cc0a058ab5..667c266539 100644 --- a/packages/xo-server/config.toml +++ b/packages/xo-server/config.toml @@ -242,6 +242,8 @@ vdiDelayBeforeRemovingCloudConfigDrive = '5 min' vdiExportConcurrency = 12 vmEvacuationConcurrency = 3 vmExportConcurrency = 2 +# The duration XO will wait for a VM to shut down cleanly before forcing it +vmShutdownTimeout = '10 minutes' vmSnapshotConcurrency = 2 poolMarkingInterval = '6 hours' diff --git a/packages/xo-server/docs/rolling-pool-update-reboot.md b/packages/xo-server/docs/rolling-pool-update-reboot.md index e18409d06c..c4ca3165b6 100644 --- a/packages/xo-server/docs/rolling-pool-update-reboot.md +++ b/packages/xo-server/docs/rolling-pool-update-reboot.md @@ -58,7 +58,7 @@ Old traces are garbage-collected on mtime (`rpu.tracesRetention`, 31 days by def | Timeout on `Waiting for host to be up` | Host takes too long to boot. `xapiOptions.restartHostTimeout` (default 20 minutes). | | Pool stays `disconnected` after the master rebooted, `EHOSTUNREACH` | Stale connection error, the retry did not kick in yet. `POST /rest/v0/servers//actions/connect` reconnects immediately. | -Note on granularity: `Evacuate` is a single `host.evacuate` XAPI call, there is no per-VM detail in the tree for that phase. Per-VM subtasks only exist in `Migrate VMs back`. +Note on granularity: `Evacuate` is a single `host.evacuate` XAPI call, there is no per-VM detail in the tree for that phase. When `shutdownPinnedVms` is enabled and pinned VMs are present, per-VM subtasks also appear under `Shut down pinned VMs` and `Restart pinned VMs`. ## Task logs @@ -70,12 +70,20 @@ Rolling pool update and rolling pool reboot task logs have major parts in common task.start({ name: 'Rolling pool reboot', poolId: string, poolName: string }) ├─ task.start({ name: 'Restarting hosts', total: number, progress: number, done: number }) | ├─ task.start({ name: `Restarting host ${hostId}`, hostId: string, hostName: string }) +| | ├─ task.start({ name: 'Shut down pinned VMs', hostId: string, hostName: string }) +| | | ├─ task.start({ name: `Shutting down VM ${vmId}`, hostId: string, hostName: string, vmId: string, vmName: string }) +│ │ │ │ └─ task.end +│ │ │ └─ task.end | | ├─ task.start({ name: 'Evacuate', hostId: string, hostName: string }) │ │ │ └─ task.end | | ├─ task.start({ name: 'Restart', hostId: string, hostName: string }) │ │ │ └─ task.end | | ├─ task.start({ name: 'Waiting for host to be up', hostId: string, hostName: string }) │ │ │ └─ task.end +| | ├─ task.start({ name: 'Restart pinned VMs', hostId: string, hostName: string }) +| | | ├─ task.start({ name: `Restarting VM ${vmId} on host ${hostId}`, hostId: string, hostName: string, vmId: string, vmName: string }) +│ │ │ │ └─ task.end +│ │ │ └─ task.end │ │ └─ task.end │ └─ task.end ├─ task.start({ name: 'Migrate VMs back' }) @@ -100,6 +108,10 @@ task.start({ name: 'Rolling pool update', poolId: string, poolName: string }) │ │ └─ task.end │ ├─ task.start({ name: 'Restarting hosts', total: number, progress: number, done: number }) │ | ├─ task.start({ name: `Restarting host ${hostId}`, hostId: string, hostName: string }) +│ | | ├─ task.start({ name: 'Shut down pinned VMs', hostId: string, hostName: string }) +│ | | | ├─ task.start({ name: `Shutting down VM ${vmId}`, hostId: string, hostName: string, vmId: string, vmName: string }) +│ │ │ │ │ └─ task.end +│ │ │ │ └─ task.end │ | | ├─ task.start({ name: 'Evacuate', hostId: string, hostName: string }) │ │ │ │ └─ task.end │ | | ├─ task.start({ name: 'Installing patches', hostId: string, hostName: string }) @@ -108,6 +120,10 @@ task.start({ name: 'Rolling pool update', poolId: string, poolName: string }) │ │ │ │ └─ task.end │ | | ├─ task.start({ name: 'Waiting for host to be up', hostId: string, hostName: string }) │ │ │ │ └─ task.end +│ | | ├─ task.start({ name: 'Restart pinned VMs', hostId: string, hostName: string }) +│ | | | ├─ task.start({ name: `Restarting VM ${vmId} on host ${hostId}`, hostId: string, hostName: string, vmId: string, vmName: string }) +│ │ │ │ │ └─ task.end +│ │ │ │ └─ task.end │ │ │ └─ task.end │ │ └─ task.end │ ├─ task.start({ name: 'Migrate VMs back' }) diff --git a/packages/xo-server/src/api/pool.mjs b/packages/xo-server/src/api/pool.mjs index cf9a803bab..20721c7bab 100644 --- a/packages/xo-server/src/api/pool.mjs +++ b/packages/xo-server/src/api/pool.mjs @@ -242,7 +242,7 @@ installPatches.description = 'Install patches on hosts' // ------------------------------------------------------------------- -export const rollingUpdate = async function ({ bypassBackupCheck = false, pool, rebootVm }) { +export const rollingUpdate = async function ({ bypassBackupCheck = false, pool, rebootVm, shutdownPinnedVms }) { const poolId = pool.id if (bypassBackupCheck) { log.warn('pool.rollingUpdate with argument "bypassBackupCheck" set to true', { poolId }) @@ -250,7 +250,7 @@ export const rollingUpdate = async function ({ bypassBackupCheck = false, pool, await backupGuard.call(this, poolId) } - await this.rollingPoolUpdate(pool, { rebootVm }) + await this.rollingPoolUpdate(pool, { rebootVm, shutdownPinnedVms }) } rollingUpdate.params = { @@ -263,6 +263,12 @@ rollingUpdate.params = { optional: true, type: 'boolean', }, + // shut down VMs that cannot be migrated (PCI passthrough, vGPU, SR-IOV VIF) + // before their host reboots and start them again on it afterwards + shutdownPinnedVms: { + default: false, + type: 'boolean', + }, } rollingUpdate.resolve = { @@ -271,7 +277,7 @@ rollingUpdate.resolve = { // ------------------------------------------------------------------- -export async function rollingReboot({ bypassBackupCheck, pool }) { +export async function rollingReboot({ bypassBackupCheck, pool, shutdownPinnedVms }) { const poolId = pool.id if (bypassBackupCheck) { log.warn('pool.rollingReboot with argument "bypassBackupCheck" set to true', { poolId }) @@ -279,7 +285,7 @@ export async function rollingReboot({ bypassBackupCheck, pool }) { await backupGuard.call(this, poolId) } - await this.rollingPoolReboot(pool) + await this.rollingPoolReboot(pool, { shutdownPinnedVms }) } rollingReboot.params = { @@ -288,6 +294,12 @@ rollingReboot.params = { type: 'boolean', }, pool: { type: 'string' }, + // shut down VMs that cannot be migrated (PCI passthrough, vGPU, SR-IOV VIF) + // before their host reboots and start them again on it afterwards + shutdownPinnedVms: { + default: false, + type: 'boolean', + }, } rollingReboot.resolve = { diff --git a/packages/xo-server/src/xapi/index.mjs b/packages/xo-server/src/xapi/index.mjs index 783ceefff5..0a571b7fe7 100644 --- a/packages/xo-server/src/xapi/index.mjs +++ b/packages/xo-server/src/xapi/index.mjs @@ -78,6 +78,7 @@ export default class Xapi extends XapiBase { vmEvacuationConcurrency, vmExportConcurrency, vmMigrationConcurrency = 3, + vmShutdownTimeout, vmSnapshotConcurrency, ...opts }) { @@ -87,6 +88,7 @@ export default class Xapi extends XapiBase { this._maxUncoalescedVdis = maxUncoalescedVdis this._restartHostTimeout = parseDuration(restartHostTimeout) this._vmEvacuationConcurrency = vmEvacuationConcurrency + this._vmShutdownTimeout = parseDuration(vmShutdownTimeout) // close event is emitted when the export is canceled via browser. See https://github.com/vatesfr/xen-orchestra/issues/5535 const waitStreamEnd = async stream => fromEvents(await stream, ['end', 'close']) diff --git a/packages/xo-server/src/xapi/mixins/patching.mjs b/packages/xo-server/src/xapi/mixins/patching.mjs index 55c031ca81..24d1c0f124 100644 --- a/packages/xo-server/src/xapi/mixins/patching.mjs +++ b/packages/xo-server/src/xapi/mixins/patching.mjs @@ -726,7 +726,11 @@ const methods = { }) }, - async rollingPoolUpdate($defer, parentTask, { xsCredentials, force = false, rebootVm = force } = {}) { + async rollingPoolUpdate( + $defer, + parentTask, + { xsCredentials, force = false, rebootVm = force, shutdownPinnedVms = false } = {} + ) { if (some(this.objects.indexes.type.SR, { type: 'linstor' })) { await this._updateLinstorPackages() } @@ -790,6 +794,7 @@ const methods = { await Task.run({ properties: { name: `Updating and rebooting` } }, async () => { await this.rollingPoolReboot(parentTask, { xsCredentials, + shutdownPinnedVms, beforeEvacuateVms: () => { // On XS < 8.4 and CH, start by installing patches on all hosts if (!isXcp && !isXsWithCdnUpdates) { diff --git a/packages/xo-server/src/xapi/mixins/pool.mjs b/packages/xo-server/src/xapi/mixins/pool.mjs index 675a5e09db..30957b5438 100644 --- a/packages/xo-server/src/xapi/mixins/pool.mjs +++ b/packages/xo-server/src/xapi/mixins/pool.mjs @@ -1,3 +1,4 @@ +import { asyncEach } from '@vates/async-each' import { cancelable, timeout } from 'promise-toolbox' import { createLogger } from '@xen-orchestra/log' import { decorateObject } from '@vates/decorate-with' @@ -14,6 +15,18 @@ const log = createLogger('xo:xapi') const PATH_DB_DUMP = '/pool/xmldbdump' +// XAPI error codes identifying VMs that can never be evacuated because they use +// a host-bound device (PCI passthrough, vGPU, SR-IOV VIF): these VMs can only +// be handled by shutting them down before their host reboots and starting them +// again on it afterwards +const PINNED_VM_ERROR_CODES = new Set(['VM_HAS_PCI_ATTACHED', 'VM_HAS_VGPU', 'VM_HAS_SRIOV_VIF']) + +// pinned VMs are shut down in parallel to keep the host's downtime short, but +// started back more conservatively to avoid a boot storm on a host which has +// just rebooted +const PINNED_VM_SHUTDOWN_CONCURRENCY = 8 +const PINNED_VM_START_CONCURRENCY = 2 + const setProgress = (task, progress) => task.set('progress', Math.round(progress)) const methods = { @@ -35,7 +48,11 @@ const methods = { }) }, - async rollingPoolReboot($defer, parentTask, { beforeEvacuateVms, beforeRebootHost, ignoreHost } = {}) { + async rollingPoolReboot( + $defer, + parentTask, + { beforeEvacuateVms, beforeRebootHost, ignoreHost, shutdownPinnedVms = false } = {} + ) { if (this.pool.ha_enabled) { const haSrs = this.pool.$ha_statefiles.map(vdi => vdi.SR) const haConfig = this.pool.ha_configuration @@ -64,9 +81,60 @@ const methods = { } } + // when shutdownPinnedVms is enabled, pinned VMs will be shut down before + // their host reboots and started again on it afterwards, otherwise their + // UUIDs are collected to raise a single actionable error covering the + // whole pool, any other evacuation blocker aborts the run + // + // this check requires HA to be already disabled: with HA enabled, XAPI + // reports every non-protected VM as an evacuation blocker + const unhandledPinnedVmUuids = [] await Promise.all( - hosts.filter(host => !ignoreHost || !ignoreHost(host)).map(host => host.$call('assert_can_evacuate')) + hosts + .filter(host => !ignoreHost || !ignoreHost(host)) + .map(async host => { + const blockedVms = await host.$call('get_vms_which_prevent_evacuation') + const vmRefs = Object.keys(blockedVms) + if (vmRefs.length === 0) { + return + } + + const canHandleAllBlockers = Object.values(blockedVms).every(([errorCode]) => + PINNED_VM_ERROR_CODES.has(errorCode) + ) + if (!canHandleAllBlockers) { + // let XAPI raise its canonical CANNOT_EVACUATE_HOST error + return host.$call('assert_can_evacuate') + } + + if (!shutdownPinnedVms) { + unhandledPinnedVmUuids.push(...vmRefs.map(vmRef => this.getObject(vmRef).uuid)) + } + }) ) + if (unhandledPinnedVmUuids.length > 0) { + // the run can proceed if the caller consents to shut these VMs down + // during their host's reboot, by enabling shutdownPinnedVms + throw incorrectState({ + actual: unhandledPinnedVmUuids, + expected: [], + object: this.pool.uuid, + property: 'pinnedVms', + }) + } + + // VMs shut down for their host's reboot and not started again yet: if the + // run aborts, leave them running rather than halted + const haltedPinnedVms = new Map() // VM ref -> host ref + $defer(async () => { + for (const [vmRef, hostRef] of haltedPinnedVms) { + try { + await this.callAsync('VM.start_on', vmRef, hostRef, false, false) + } catch (error) { + log.warn('failed to restart pinned VM after an aborted rolling pool reboot', { vmRef, error }) + } + } + }) // Steps in the RPR : Evacuate hosts, reboot hosts, migrate VMs back, and potentially updateHosts (beforeEvacuateVms and beforeRebootHost) const nSteps = 3 + Number(beforeEvacuateVms !== undefined) + Number(beforeRebootHost !== undefined) @@ -130,6 +198,42 @@ const methods = { const getServerTime = async () => parseDateTime(await this.call('host.get_servertime', host.$ref)) * 1e3 + let pinnedVmRefs = [] + if (shutdownPinnedVms) { + // fresh query instead of reusing the initial check: the pool + // state may have changed while handling the previous hosts + const blockedVms = await host.$call('get_vms_which_prevent_evacuation') + pinnedVmRefs = Object.entries(blockedVms) + .filter(([, [errorCode]]) => PINNED_VM_ERROR_CODES.has(errorCode)) + .map(([vmRef]) => vmRef) + + if (pinnedVmRefs.length > 0) { + await Task.run({ properties: { name: `Shut down pinned VMs`, hostId, hostName } }, async () => { + await asyncEach( + pinnedVmRefs, + async vmRef => { + const { uuid: vmId, name_label: vmName } = this.getObject(vmRef) + await Task.run( + { properties: { name: `Shutting down VM ${vmId}`, hostId, hostName, vmId, vmName } }, + async () => { + try { + // a guest may ignore the shutdown request: cancel it and force the shutdown + // instead of blocking the whole run, the user consented to these VMs going down + await timeout.call(this.callAsync('VM.clean_shutdown', vmRef), this._vmShutdownTimeout) + } catch (error) { + log.warn('clean shutdown of a pinned VM failed, forcing it', { vmId, error }) + await this.callAsync('VM.hard_shutdown', vmRef) + } + } + ) + haltedPinnedVms.set(vmRef, host.$ref) + }, + { concurrency: PINNED_VM_SHUTDOWN_CONCURRENCY, stopOnError: true } + ) + }) + } + } + // the pool state may have changed since the initial check, e.g. while evacuating the previous hosts await Task.run({ properties: { name: `Check evacuation precondition`, hostId, hostName } }, async () => { await host.$call('assert_can_evacuate') @@ -194,6 +298,26 @@ const methods = { new Error(`Host ${hostId} took too long to restart`) ) }) + + if (pinnedVmRefs.length > 0) { + await Task.run({ properties: { name: `Restart pinned VMs`, hostId, hostName } }, async () => { + // stopOnError: still try to start every pinned VM of this host before failing the run + await asyncEach( + pinnedVmRefs, + async vmRef => { + const { uuid: vmId, name_label: vmName } = this.getObject(vmRef) + await Task.run( + { + properties: { name: `Restarting VM ${vmId} on host ${hostId}`, hostId, hostName, vmId, vmName }, + }, + () => this.callAsync('VM.start_on', vmRef, host.$ref, false, false) + ) + haltedPinnedVms.delete(vmRef) + }, + { concurrency: PINNED_VM_START_CONCURRENCY, stopOnError: false } + ) + }) + } rprProgress += progressStepPerHost setProgress(parentTask, rprProgress) subtaskProgress += subtaskProgressStep diff --git a/packages/xo-server/src/xo-mixins/pool.mjs b/packages/xo-server/src/xo-mixins/pool.mjs index b631e1d5b8..1e5ce11213 100644 --- a/packages/xo-server/src/xo-mixins/pool.mjs +++ b/packages/xo-server/src/xo-mixins/pool.mjs @@ -205,7 +205,7 @@ export default class Pools { ) } - async rollingPoolReboot(pool, { parentTask } = {}) { + async rollingPoolReboot(pool, { parentTask, shutdownPinnedVms } = {}) { const { _app } = this await _app.checkFeatureAuthorization('ROLLING_POOL_REBOOT') const releaseGuard = acquireRpuGuard(pool.id, 'rollingPoolReboot') @@ -226,7 +226,7 @@ export default class Pools { } const task = parentTask === undefined ? _app.tasks.create(properties) : new Task({ properties }) trace?.attach(task) - await task.run(async () => _app.getXapi(pool).rollingPoolReboot(task)) + await task.run(async () => _app.getXapi(pool).rollingPoolReboot(task, { shutdownPinnedVms })) } finally { trace?.stop() releaseGuard() diff --git a/packages/xo-server/src/xo-mixins/xen-servers.mjs b/packages/xo-server/src/xo-mixins/xen-servers.mjs index 11225d76f9..34a2a3c75d 100644 --- a/packages/xo-server/src/xo-mixins/xen-servers.mjs +++ b/packages/xo-server/src/xo-mixins/xen-servers.mjs @@ -900,7 +900,7 @@ export default class XenServers { }) } - async rollingPoolUpdate($defer, pool, { rebootVm, parentTask } = {}) { + async rollingPoolUpdate($defer, pool, { rebootVm, parentTask, shutdownPinnedVms } = {}) { const app = this._app await app.checkFeatureAuthorization('ROLLING_POOL_UPDATE') const [schedules, jobs] = await Promise.all([app.getAllSchedules(), app.getAllJobs('backup')]) @@ -973,6 +973,7 @@ export default class XenServers { this.getXapi(pool).rollingPoolUpdate(task, { xsCredentials: app.apiContext.user.preferences.xsCredentials, rebootVm, + shutdownPinnedVms, }) ) } diff --git a/packages/xo-web/src/common/intl/messages.js b/packages/xo-web/src/common/intl/messages.js index 3cb75d7c58..74452cb979 100644 --- a/packages/xo-web/src/common/intl/messages.js +++ b/packages/xo-web/src/common/intl/messages.js @@ -2689,6 +2689,8 @@ const messages = { replicationCountHigherThanHostsWithDisks: 'Replication count is higher than number of hosts with disks', resourceList: 'Resource list', rpuRequireVmsReboot: 'To fully apply the patches, some VMs will reboot. Are you sure you want to continue?', + rpuShutdownPinnedVms: + 'The following VMs use a host-bound device (PCI passthrough, vGPU, SR-IOV VIFs) and cannot be migrated. They will be shut down before their host reboots and started again on it afterwards. Are you sure you want to continue?', selectDisks: 'Select disk(s)…', selectedDiskTypeIncompatibleXostor: 'Only disks of type "Disk" and "Raid" are accepted. Selected disk type: {type}.', setAsPreferred: 'Set as preferred', diff --git a/packages/xo-web/src/common/xo/index.js b/packages/xo-web/src/common/xo/index.js index 66aa1b42be..0006e4e810 100644 --- a/packages/xo-web/src/common/xo/index.js +++ b/packages/xo-web/src/common/xo/index.js @@ -998,23 +998,45 @@ export const rollingPoolReboot = async pool => { title: _('rollingPoolReboot'), icon: 'pool-rolling-reboot', }) - try { - return await _call('pool.rollingReboot', { pool: poolId }) - } catch (error) { - if (!forbiddenOperation.is(error)) { + const rpr = async ({ bypassBackupCheck = false, shutdownPinnedVms = false } = {}) => { + try { + return await _call('pool.rollingReboot', { pool: poolId, bypassBackupCheck, shutdownPinnedVms }) + } catch (error) { + if (forbiddenOperation.is(error)) { + await confirm({ + body: ( +

+ {_('bypassBackupPoolModalMessage')} +

+ ), + title: _('rollingPoolReboot'), + icon: 'pool-rolling-reboot', + }) + return rpr({ bypassBackupCheck: true, shutdownPinnedVms }) + } + if (incorrectState.is(error, { property: 'pinnedVms' })) { + await confirm({ + body: ( +
+

+ {_('rpuShutdownPinnedVms')} +

+
    + {error.data.actual.map(vmId => ( +
  • {renderXoItemFromId(vmId)}
    • + ))} +
+
+ ), + title: _('rollingPoolReboot'), + icon: 'pool-rolling-reboot', + }) + return rpr({ bypassBackupCheck, shutdownPinnedVms: true }) + } throw error } - await confirm({ - body: ( -

- {_('bypassBackupPoolModalMessage')} -

- ), - title: _('rollingPoolReboot'), - icon: 'pool-rolling-reboot', - }) - return _call('pool.rollingReboot', { pool: poolId, bypassBackupCheck: true }) } + return rpr() } export const getPoolGuestSecureBootReadiness = async poolId => { @@ -1433,9 +1455,9 @@ export const rollingPoolUpdate = async poolId => { icon: 'pool-rolling-update', }) - const rpu = async ({ bypassBackupCheck = false, rebootVm = false } = {}) => { + const rpu = async ({ bypassBackupCheck = false, rebootVm = false, shutdownPinnedVms = false } = {}) => { try { - await _call('pool.rollingUpdate', { pool: poolId, bypassBackupCheck, rebootVm }) + await _call('pool.rollingUpdate', { pool: poolId, bypassBackupCheck, rebootVm, shutdownPinnedVms }) subscribeHostMissingPatches.forceRefresh() } catch (err) { if (forbiddenOperation.is(err)) { @@ -1448,7 +1470,7 @@ export const rollingPoolUpdate = async poolId => { title: _('rollingPoolUpdate'), icon: 'pool-rolling-update', }) - await rpu({ bypassBackupCheck: true, rebootVm }) + return rpu({ bypassBackupCheck: true, rebootVm, shutdownPinnedVms }) } if (incorrectState.is(err, { property: 'guidance' })) { await confirm({ @@ -1460,8 +1482,28 @@ export const rollingPoolUpdate = async poolId => { title: _('rollingPoolUpdate'), icon: 'pool-rolling-update', }) - await rpu({ bypassBackupCheck, rebootVm: true }) + return rpu({ bypassBackupCheck, rebootVm: true, shutdownPinnedVms }) } + if (incorrectState.is(err, { property: 'pinnedVms' })) { + await confirm({ + body: ( +
+

+ {_('rpuShutdownPinnedVms')} +

+
    + {err.data.actual.map(vmId => ( +
  • {renderXoItemFromId(vmId)}
    • + ))} +
+
+ ), + title: _('rollingPoolUpdate'), + icon: 'pool-rolling-update', + }) + return rpu({ bypassBackupCheck, rebootVm, shutdownPinnedVms: true }) + } + throw err } }