diff --git a/.gitignore b/.gitignore index 01a5ca4588..add9cd21d9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /@vates/*/dist/ /@vates/*/node_modules/ +/@vates/node-vsphere-soap/test/config-test.mjs /@xen-orchestra/*/dist/ /@xen-orchestra/*/node_modules/ /@xen-orchestra/rest-api/open-api/spec/swagger.json diff --git a/@vates/node-vsphere-soap/lib/_parseFault.mjs b/@vates/node-vsphere-soap/lib/_parseFault.mjs new file mode 100644 index 0000000000..9db70ea294 --- /dev/null +++ b/@vates/node-vsphere-soap/lib/_parseFault.mjs @@ -0,0 +1,102 @@ +const XML_ENTITIES = { + '&': '&', + ''': "'", + '>': '>', + '<': '<', + '"': '"', +} + +const unescapeXml = text => text.replace(/&(?:amp|apos|gt|lt|quot);/g, entity => XML_ENTITIES[entity]) + +// node-soap wraps text nodes either as a plain string or as `{ $value }` depending on +// whether the element carries attributes +const valueOf = node => { + if (typeof node === 'string') { + return node + } + if (node !== null && typeof node === 'object' && typeof node.$value === 'string') { + return node.$value + } +} + +const faultTypeOfElementName = name => name.replace(/^.*:/, '').replace(/Fault$/, '') + +// SOAP 1.1 requires the children of `Fault` to be unqualified, and that is what ESXi and vCenter +// send, but the raw body is only read when node-soap failed to parse the response: it can then be +// anything, e.g. an error emitted by a reverse proxy in front of vCenter. Hence the tolerance to a +// namespace prefix and to attributes on the element +const textOfElement = name => + new RegExp(`<(?:[\\w.-]+:)?${name}(?:\\s[^>]*)?>([\\s\\S]*?)`, 'i') + +const FAULTCODE_RE = textOfElement('faultcode') +const FAULTSTRING_RE = textOfElement('faultstring') +const LOCALIZED_MESSAGE_RE = textOfElement('localizedMessage') +const DETAIL_CHILD_RE = /<(?:[\w.-]+:)?detail(?:\s[^>]*)?>\s*<([^\s/>]+)/i + +/** + * Extracts the vim25 fault of a failed SOAP call. + * + * The fault type (`FileLocked`, `InvalidDeviceSpec`, `ManagedObjectNotFound`, ...) is the only part + * a caller can branch on: `faultstring` and `localizedMessage` are meant for humans and change with + * the locale of the server. + * + * Two sources are used, in order of reliability: + * 1. `rawError.root`, the envelope parsed by node-soap ( see `wsdl.xmlToObject`, which throws on a + * fault and whose error is stored there by node-soap) + * 2. the raw response body, when the parsing itself failed + * + * @param {unknown} rawError - the error passed by node-soap to its callback + * @returns {{ code?: string, faultcode?: string, faultstring?: string, localizedMessage?: string, body?: string }} + */ +export function parseFault(rawError) { + const fault = rawError?.root?.Envelope?.Body?.Fault + const rawBody = rawError?.response?.data ?? rawError?.body + const body = typeof rawBody === 'string' && rawBody !== '' ? rawBody : undefined + + let code, localizedMessage + let faultcode = valueOf(fault?.faultcode) + let faultstring = valueOf(fault?.faultstring) + + const detail = fault?.detail + if (detail !== null && typeof detail === 'object') { + // the single child of `detail` names the fault, e.g. + // `` + for (const [name, value] of Object.entries(detail)) { + if (name === 'attributes') { + continue + } + code = value?.attributes?.['xsi:type'] ?? faultTypeOfElementName(name) + localizedMessage = valueOf(value?.localizedMessage) + break + } + } + + if (body !== undefined) { + if (faultstring === undefined) { + const matches = body.match(FAULTSTRING_RE) + if (matches !== null) { + faultstring = unescapeXml(matches[1]) + } + } + if (faultcode === undefined) { + const matches = body.match(FAULTCODE_RE) + if (matches !== null) { + faultcode = unescapeXml(matches[1]) + } + } + if (code === undefined) { + const matches = body.match(DETAIL_CHILD_RE) + if (matches !== null) { + code = faultTypeOfElementName(matches[1]) + } + } + if (localizedMessage === undefined) { + const matches = body.match(LOCALIZED_MESSAGE_RE) + if (matches !== null) { + localizedMessage = unescapeXml(matches[1]) + } + } + } + + return { code, faultcode, faultstring, localizedMessage, body } +} diff --git a/@vates/node-vsphere-soap/lib/_parseFault.test.mjs b/@vates/node-vsphere-soap/lib/_parseFault.test.mjs new file mode 100644 index 0000000000..7ab4637090 --- /dev/null +++ b/@vates/node-vsphere-soap/lib/_parseFault.test.mjs @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { parseFault } from './_parseFault.mjs' + +// captured from an ESXi 8 host +const MANAGED_OBJECT_NOT_FOUND = ` + + +ServerFaultCodeThe object 'vim.VirtualMachine:vm-42' has already been deleted or has not been completely createdvm-42 + +` + +// not what a vSphere host sends — SOAP 1.1 requires these children to be unqualified — but a body +// which node-soap failed to parse can come from anything sitting in front of vCenter +const NAMESPACE_PREFIXED = ` + + +ServerFaultCodeUnable to access file since it is lockedUnable to access file [ds1] vm/vm.vmdk since it is locked + +` + +describe('parseFault', function () { + it('reads the fault from the envelope parsed by node-soap', function () { + const { code, faultcode, faultstring, localizedMessage } = parseFault({ + root: { + Envelope: { + Body: { + Fault: { + faultcode: 'ServerFaultCode', + faultstring: 'Unable to access file since it is locked', + detail: { + FileLockedFault: { + attributes: { 'xsi:type': 'FileLocked' }, + localizedMessage: 'Unable to access file [ds1] vm/vm.vmdk since it is locked', + }, + }, + }, + }, + }, + }, + }) + + assert.equal(code, 'FileLocked') + assert.equal(faultcode, 'ServerFaultCode') + assert.equal(faultstring, 'Unable to access file since it is locked') + assert.equal(localizedMessage, 'Unable to access file [ds1] vm/vm.vmdk since it is locked') + }) + + it('falls back to the element name when xsi:type is absent', function () { + const { code } = parseFault({ + root: { Envelope: { Body: { Fault: { detail: { InvalidDeviceSpecFault: {} } } } } }, + }) + + assert.equal(code, 'InvalidDeviceSpec') + }) + + it('falls back to the raw body when the envelope could not be parsed', function () { + const { code, faultcode, faultstring, body } = parseFault({ + message: 'Request failed with status code 500', + response: { data: MANAGED_OBJECT_NOT_FOUND }, + }) + + assert.equal(code, 'ManagedObjectNotFound') + assert.equal(faultcode, 'ServerFaultCode') + // entities are decoded + assert.match(faultstring, /^The object 'vim\.VirtualMachine:vm-42' has already been deleted/) + assert.equal(body, MANAGED_OBJECT_NOT_FOUND) + }) + + it('reads a namespace prefixed fault from the raw body', function () { + const { code, faultcode, faultstring, localizedMessage } = parseFault({ + response: { data: NAMESPACE_PREFIXED }, + }) + + assert.equal(code, 'FileLocked') + assert.equal(faultcode, 'ServerFaultCode') + assert.equal(faultstring, 'Unable to access file since it is locked') + assert.equal(localizedMessage, 'Unable to access file [ds1] vm/vm.vmdk since it is locked') + }) + + it('handles a `$value` wrapped text node', function () { + const { faultstring } = parseFault({ + root: { Envelope: { Body: { Fault: { faultstring: { $value: 'boom' } } } } }, + }) + + assert.equal(faultstring, 'boom') + }) + + it('does not throw on an error without any fault', function () { + assert.deepEqual(parseFault(new Error('socket hang up')), { + code: undefined, + faultcode: undefined, + faultstring: undefined, + localizedMessage: undefined, + body: undefined, + }) + assert.deepEqual(parseFault(undefined).code, undefined) + }) + + it('ignores the placeholder body used by node-soap for streamed responses', function () { + assert.equal(parseFault({ body: '' }).body, undefined) + }) +}) diff --git a/@vates/node-vsphere-soap/lib/client.mjs b/@vates/node-vsphere-soap/lib/client.mjs index 4b5e778566..09058d765e 100644 --- a/@vates/node-vsphere-soap/lib/client.mjs +++ b/@vates/node-vsphere-soap/lib/client.mjs @@ -19,16 +19,29 @@ import soap from 'soap' import Cookie from 'soap-cookie' // required for session persistence import { createLogger } from '@xen-orchestra/log' +import { parseFault } from './_parseFault.mjs' + +// exposed for the consumers talking to the host outside of the WSDL, e.g. `Fetch` +export { parseFault } from './_parseFault.mjs' + const { warn } = createLogger('xo:node-vsphere-soap:client') class VmwareError extends Error { constructor(rawError) { - super(rawError.body) - const matches = rawError.body?.match(/]*>(.*)<\/faultstring>/im) - this.message = matches?.[1] ?? rawError.message + const { code, faultcode, faultstring, localizedMessage, body } = parseFault(rawError) + super(localizedMessage ?? faultstring ?? rawError?.message ?? 'unknown SOAP error') this.name = 'VmwareError' - this.stack = rawError.stack - // not putting the cause since the error failed to be stringified correctly down the road - warn(rawError) + // the vim25 fault type, the only part of a fault a caller can branch on + this.code = code + this.faultcode = faultcode + this.faultstring = faultstring + this.localizedMessage = localizedMessage + this.body = body + if (rawError?.stack !== undefined) { + this.stack = rawError.stack + } + // legacy constraint: not putting the cause since the error failed to be stringified correctly + // down the road, the fault is exposed through own properties instead + warn('SOAP call failed', { code, faultcode, faultstring, localizedMessage }) } } // Client class @@ -38,6 +51,7 @@ class VmwareError extends Error { export function Client(vCenterHostname, username, password, sslVerify) { this.status = 'disconnected' this.reconnectCount = 0 + this._exitHook = undefined sslVerify = typeof sslVerify !== 'undefined' ? sslVerify : false @@ -90,44 +104,49 @@ util.inherits(Client, EventEmitter) Client.prototype.runCommand = function (command, args) { const self = this - let cmdargs - if (!args || args === null) { - cmdargs = {} - } else { - cmdargs = args - } + const cmdargs = args ?? {} const emitter = new EventEmitter() + const onResponse = function (err, result, raw, soapHeader) { + if (err) { + // an error and a result are mutually exclusive: emitting both makes every caller listening + // for 'result' believe the call succeeded, with `result` undefined + _soapErrorHandler(self, emitter, command, cmdargs, err) + return + } + if (command === 'Logout') { + self.status = 'disconnected' + self._unregisterExitHook() + } + emitter.emit('result', result, raw, soapHeader) + } + + const send = function () { + self.client.VimService.VimPort[command](cmdargs, onResponse) + } + // check if client has successfully connected if (self.status === 'ready' || self.status === 'connecting') { - self.client.VimService.VimPort[command](cmdargs, function (err, result, raw, soapHeader) { - if (err) { - _soapErrorHandler(self, emitter, command, cmdargs, err) - } - if (command === 'Logout') { - self.status = 'disconnected' - process.removeAllListeners('beforeExit') - } - emitter.emit('result', result, raw, soapHeader) - }) + send() } else { // if connection not ready or connecting, reconnect to instance if (self.status === 'disconnected') { self.emit('connect') } - self.once('ready', function () { - self.client.VimService.VimPort[command](cmdargs, function (err, result, raw, soapHeader) { - if (err) { - _soapErrorHandler(self, emitter, command, cmdargs, err) - } - if (command === 'Logout') { - self.status = 'disconnected' - process.removeAllListeners('beforeExit') - } - emitter.emit('result', result, raw, soapHeader) - }) - }) + + // a failed connection would otherwise leave this listener registered for ever, and the next + // successful one would replay every command whose caller has long given up + const onConnectionError = error => { + self.off('ready', onReady) + emitter.emit('error', error) + } + const onReady = () => { + self.off('error', onConnectionError) + send() + } + self.once('ready', onReady) + self.once('error', onConnectionError) } return emitter @@ -153,9 +172,11 @@ Client.prototype._connect = function () { self.clientopts, function (err, client) { if (err) { - const vErr = new VmwareError(err) - self.emit('error', vErr) - throw vErr + self.status = 'disconnected' + // throwing here would be an uncaught exception: we are in an async callback, not in the + // stack of the caller + self.emit('error', new VmwareError(err)) + return } self.client = client // save client for later use @@ -186,7 +207,7 @@ Client.prototype._connect = function () { self.status = 'ready' self.emit('ready') - process.once('beforeExit', self._close) + self._registerExitHook() }) .once('error', function (err) { self.status = 'disconnected' @@ -202,9 +223,36 @@ Client.prototype._connect = function () { ) } +// Logs out on process exit, so that a session is not left open on the server until it expires. +// +// The hook must be stored: passing `self._close` directly would call it with `this === process`, +// and removing it with `process.removeAllListeners('beforeExit')` would drop the hooks of every +// other module of the process. +Client.prototype._registerExitHook = function () { + const self = this + + if (self._exitHook === undefined) { + self._exitHook = function () { + self._close() + } + process.once('beforeExit', self._exitHook) + } +} + +Client.prototype._unregisterExitHook = function () { + const self = this + + if (self._exitHook !== undefined) { + process.off('beforeExit', self._exitHook) + self._exitHook = undefined + } +} + Client.prototype._close = function () { const self = this + self._unregisterExitHook() + if (self.status === 'ready') { self .runCommand('Logout', { _this: self.sessionManager }) @@ -220,34 +268,64 @@ Client.prototype._close = function () { } } -function _soapErrorHandler(self, emitter, command, args, err) { +// Errors are only emitted on `emitter`, the emitter of the failed call: throwing from here would +// be an uncaught exception since we are always in an async callback, and emitting on the client +// would reject unrelated concurrent calls. +// +// Exported for the unit tests: building a `Client` connects to a host. +export function _soapErrorHandler(self, emitter, command, args, err) { err = err || { body: 'general error' } - if (err.body?.match(/session is not authenticated/)) { - self.status = 'disconnected' - process.removeAllListeners('beforeExit') + const vErr = new VmwareError(err) - if (self.reconnectCount < 10) { - self.reconnectCount += 1 - self - .runCommand(command, args) - .once('result', function (result, raw, soapHeader) { - emitter.emit('result', result, raw, soapHeader) - }) - .once('error', function (err) { - const vErr = new VmwareError(err) - emitter.emit('error', vErr) - throw vErr - }) - } else { - const vErr = new VmwareError(err) - emitter.emit('error', vErr) - throw vErr - } - } else { - const vErr = new VmwareError(err) + // `NotAuthenticated` is raised when the session expired: log in again and replay the call. + // + // The fault type is used instead of the previous `err.body?.match(...)`: node-soap only fills + // `body` for streamed responses, so the string test never matched and the session was never + // renewed + const isSessionExpired = + vErr.code === 'NotAuthenticated' || /session is not authenticated/i.test(vErr.faultstring ?? vErr.message) + + if (!isSessionExpired || self.reconnectCount >= 10) { emitter.emit('error', vErr) - throw vErr + return + } + + // The call is replayed only once a new session is ready. Handing it straight to `runCommand` + // would send it right away while the status is 'connecting' — i.e. on the very session which + // just expired — and the failure of that replay would land back here. + const onConnectionError = function (error) { + self.off('ready', onReady) + emitter.emit('error', error) + } + const onReady = function () { + self.off('error', onConnectionError) + self + .runCommand(command, args) + .once('result', function (result, raw, soapHeader) { + emitter.emit('result', result, raw, soapHeader) + }) + .once('error', function (retryError) { + emitter.emit('error', retryError) + }) + } + self.once('ready', onReady) + self.once('error', onConnectionError) + + // A whole wave of calls can fail on the same expired session, e.g. during an import. Only the + // first one to notice drops it and pays for a retry: the others ride the reconnect it started, + // instead of each burning one of the 10 and of resetting the status under a `_connect` already + // in flight — which used to start a second `soap.createClient` and replace `self.client` under + // the first one. + if (self.status === 'ready') { + self.status = 'disconnected' + self._unregisterExitHook() + self.reconnectCount += 1 + } + if (self.status === 'disconnected') { + // no reconnect in flight: either this call is the first to notice, or the previous attempt + // already failed + self.emit('connect') } } diff --git a/@vates/node-vsphere-soap/lib/client.test.mjs b/@vates/node-vsphere-soap/lib/client.test.mjs new file mode 100644 index 0000000000..b439c69e31 --- /dev/null +++ b/@vates/node-vsphere-soap/lib/client.test.mjs @@ -0,0 +1,219 @@ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' +import { describe, it } from 'node:test' + +import { _soapErrorHandler } from './client.mjs' + +// what a host answers to any call made on a session which expired +const NOT_AUTHENTICATED = { + root: { + Envelope: { + Body: { + Fault: { + faultcode: 'ServerFaultCode', + faultstring: 'The session is not authenticated.', + detail: { NotAuthenticatedFault: { attributes: { 'xsi:type': 'NotAuthenticated' } } }, + }, + }, + }, + }, +} + +const FILE_LOCKED = { + root: { + Envelope: { + Body: { + Fault: { + faultcode: 'ServerFaultCode', + faultstring: 'Unable to access file since it is locked', + detail: { FileLockedFault: { attributes: { 'xsi:type': 'FileLocked' } } }, + }, + }, + }, + }, +} + +/** + * A stand-in for a connected `Client`, reduced to what `_soapErrorHandler` touches: building a real + * one talks to a host. + * + * `connect` reproduces `_connect`, which only starts connecting from 'disconnected'. + */ +function fakeClient() { + const self = new EventEmitter() + + self.status = 'ready' + self.reconnectCount = 0 + self.unregisterExitHookCalls = 0 + self._unregisterExitHook = () => { + self.unregisterExitHookCalls += 1 + } + + // the calls handed to `runCommand`, i.e. the ones actually sent to the host + self.sent = [] + self.runCommand = (command, args) => { + const emitter = new EventEmitter() + self.sent.push({ args, command, emitter, statusWhenSent: self.status }) + return emitter + } + + self.connectCalls = 0 + self.on('connect', () => { + self.connectCalls += 1 + if (self.status === 'disconnected') { + self.status = 'connecting' + } + }) + + return self +} + +// makes one in-flight call fail, and reports what its own emitter received +function failingCall(self, command, rawError = NOT_AUTHENTICATED) { + const emitter = new EventEmitter() + const outcome = {} + emitter.once('result', result => { + outcome.result = result + }) + emitter.once('error', error => { + outcome.error = error + }) + + _soapErrorHandler(self, emitter, command, { _this: command }, rawError) + + return outcome +} + +describe('_soapErrorHandler', function () { + it('replays a call made on an expired session, once the new session is ready', function () { + const self = fakeClient() + + const outcome = failingCall(self, 'RetrievePropertiesEx') + + assert.equal(self.reconnectCount, 1) + assert.equal(self.status, 'connecting') + assert.equal(self.unregisterExitHookCalls, 1) + assert.deepEqual(self.sent, []) + + self.status = 'ready' + self.emit('ready') + + assert.equal(self.sent.length, 1) + assert.equal(self.sent[0].command, 'RetrievePropertiesEx') + self.sent[0].emitter.emit('result', 'replayed') + assert.equal(outcome.result, 'replayed') + }) + + it('starts a single reconnect for a whole wave of calls on the same session', function () { + const self = fakeClient() + + const outcomes = [failingCall(self, 'A'), failingCall(self, 'B'), failingCall(self, 'C')] + + // one retry for the wave, not one per call: they used to exhaust the 10 of a busy import + assert.equal(self.reconnectCount, 1) + assert.equal(self.connectCalls, 1) + assert.equal(self.unregisterExitHookCalls, 1) + // and none of them is sent while the reconnect is in flight, which would mean sending it on + // the session which just expired + assert.deepEqual(self.sent, []) + + self.status = 'ready' + self.emit('ready') + + assert.deepEqual( + self.sent.map(({ command, statusWhenSent }) => [command, statusWhenSent]), + [ + ['A', 'ready'], + ['B', 'ready'], + ['C', 'ready'], + ] + ) + + // each replay answers its own caller + self.sent[1].emitter.emit('result', 'B replayed') + assert.equal(outcomes[1].result, 'B replayed') + assert.equal(outcomes[0].result, undefined) + assert.equal(outcomes[2].result, undefined) + }) + + it('restarts a reconnect which already failed', function () { + const self = fakeClient() + + failingCall(self, 'A') + assert.equal(self.connectCalls, 1) + + // the reconnect failed, nothing is in flight any more + self.status = 'disconnected' + + failingCall(self, 'B') + assert.equal(self.connectCalls, 2) + // the session was already dropped by the first call, the second one must not pay for it again + assert.equal(self.reconnectCount, 1) + }) + + it('reports a failed reconnect to every waiting call, and stops listening', function () { + const self = fakeClient() + + const outcomes = [failingCall(self, 'A'), failingCall(self, 'B')] + + const connectionError = new Error('host unreachable') + self.status = 'disconnected' + self.emit('error', connectionError) + + assert.equal(outcomes[0].error, connectionError) + assert.equal(outcomes[1].error, connectionError) + assert.deepEqual(self.sent, []) + // a listener left behind would replay a call whose caller has long given up + assert.equal(self.listenerCount('ready'), 0) + assert.equal(self.listenerCount('error'), 0) + }) + + it('reports the error of a replay which failed too', function () { + const self = fakeClient() + + const outcome = failingCall(self, 'A') + + self.status = 'ready' + self.emit('ready') + + const replayError = new Error('failed again') + self.sent[0].emitter.emit('error', replayError) + assert.equal(outcome.error, replayError) + }) + + it('gives up after 10 reconnects', function () { + const self = fakeClient() + self.reconnectCount = 10 + + const outcome = failingCall(self, 'A') + + assert.equal(outcome.error.code, 'NotAuthenticated') + assert.equal(self.connectCalls, 0) + assert.equal(self.status, 'ready') + assert.deepEqual(self.sent, []) + }) + + it('emits an error which is not a session expiry as is, without reconnecting', function () { + const self = fakeClient() + + const outcome = failingCall(self, 'A', FILE_LOCKED) + + assert.equal(outcome.error.code, 'FileLocked') + assert.equal(self.connectCalls, 0) + assert.equal(self.reconnectCount, 0) + assert.equal(self.status, 'ready') + assert.deepEqual(self.sent, []) + }) + + it('recognizes an expiry which only says so in its faultstring', function () { + const self = fakeClient() + + // no `detail`, hence no fault type to branch on + failingCall(self, 'A', { + root: { Envelope: { Body: { Fault: { faultstring: 'The session is not authenticated.' } } } }, + }) + + assert.equal(self.reconnectCount, 1) + assert.equal(self.connectCalls, 1) + }) +}) diff --git a/@vates/node-vsphere-soap/package.json b/@vates/node-vsphere-soap/package.json index 06f09b5cf2..4e857839de 100644 --- a/@vates/node-vsphere-soap/package.json +++ b/@vates/node-vsphere-soap/package.json @@ -31,6 +31,7 @@ "node": ">=14" }, "scripts": { - "postversion": "npm publish --access public" + "postversion": "npm publish --access public", + "test": "node --test lib/*.test.mjs" } } diff --git a/@vates/node-vsphere-soap/test/config-test.stub.mjs b/@vates/node-vsphere-soap/test/config-test.stub.mjs index 7572fc4574..23fdb6e917 100644 --- a/@vates/node-vsphere-soap/test/config-test.stub.mjs +++ b/@vates/node-vsphere-soap/test/config-test.stub.mjs @@ -1,7 +1,7 @@ -// place your own credentials here for a vCenter or ESXi server -// this information will be used for connecting to a vCenter instance -// for module testing -// name the file config-test.js +// place your own credentials here for a vCenter or ESXi server: they are used by +// vsphere-soap.integ.mjs to connect to a real instance. +// +// copy this file next to it, as `config-test.mjs`. It is ignored by git, do not commit it. export const vCenterTestCreds = { vCenterIP: 'vcsa', diff --git a/@vates/node-vsphere-soap/test/vsphere-soap.test.mjs b/@vates/node-vsphere-soap/test/vsphere-soap.test.mjs deleted file mode 100644 index ed344ce5c0..0000000000 --- a/@vates/node-vsphere-soap/test/vsphere-soap.test.mjs +++ /dev/null @@ -1,138 +0,0 @@ -/* - vsphere-soap.test.js - - tests for the vCenterConnectionInstance class -*/ - -import assert from 'assert' -import { describe, it } from 'node:test' - -import * as vc from '../lib/client.mjs' - -// eslint-disable-next-line n/no-missing-import -import { vCenterTestCreds as TestCreds } from '../config-test.mjs' - -const VItest = new vc.Client(TestCreds.vCenterIP, TestCreds.vCenterUser, TestCreds.vCenterPassword, false) - -describe('Client object initialization:', function () { - it('provides a successful login', { timeout: 5000 }, function (t, done) { - VItest.once('ready', function () { - assert.notEqual(VItest.userName, null) - assert.notEqual(VItest.fullName, null) - assert.notEqual(VItest.serviceContent, null) - done() - }).once('error', function (err) { - console.error(err) - // this should fail if there's a problem - assert.notEqual(VItest.userName, null) - assert.notEqual(VItest.fullName, null) - assert.notEqual(VItest.serviceContent, null) - done() - }) - }) -}) - -describe('Client reconnection test:', function () { - it('can successfully reconnect', { timeout: 5000 }, function (t, done) { - VItest.runCommand('Logout', { _this: VItest.serviceContent.sessionManager }) - .once('result', function (result) { - // now we're logged out, so let's try running a command to test automatic re-login - VItest.runCommand('CurrentTime', { _this: 'ServiceInstance' }) - .once('result', function (result) { - assert(result.returnval instanceof Date) - done() - }) - .once('error', function (err) { - console.error(err) - }) - }) - .once('error', function (err) { - console.error(err) - }) - }) -}) - -// these tests don't work yet -describe('Client tests - query commands:', function () { - it('retrieves current time', { timeout: 5000 }, function (t, done) { - VItest.runCommand('CurrentTime', { _this: 'ServiceInstance' }).once('result', function (result) { - assert(result.returnval instanceof Date) - done() - }) - }) - - it('retrieves current time 2 (check for event clobbering)', { timeout: 5000 }, function (t, done) { - VItest.runCommand('CurrentTime', { _this: 'ServiceInstance' }).once('result', function (result) { - assert(result.returnval instanceof Date) - done() - }) - }) - - it('can obtain the names of all Virtual Machines in the inventory', { timeout: 20000 }, function (t, done) { - // get property collector - const propertyCollector = VItest.serviceContent.propertyCollector - // get view manager - const viewManager = VItest.serviceContent.viewManager - // get root folder - const rootFolder = VItest.serviceContent.rootFolder - - let containerView, objectSpec, traversalSpec, propertySpec, propertyFilterSpec - // this is the equivalent to - VItest.runCommand('CreateContainerView', { - _this: viewManager, - container: rootFolder, - type: ['VirtualMachine'], - recursive: true, - }).once('result', function (result) { - // build all the data structures needed to query all the vm names - containerView = result.returnval - - objectSpec = { - attributes: { 'xsi:type': 'ObjectSpec' }, // setting attributes xsi:type is important or else the server may mis-recognize types! - obj: containerView, - skip: true, - } - - traversalSpec = { - attributes: { 'xsi:type': 'TraversalSpec' }, - name: 'traverseEntities', - type: 'ContainerView', - path: 'view', - skip: false, - } - - objectSpec = { ...objectSpec, selectSet: [traversalSpec] } - - propertySpec = { - attributes: { 'xsi:type': 'PropertySpec' }, - type: 'VirtualMachine', - pathSet: ['name'], - } - - propertyFilterSpec = { - attributes: { 'xsi:type': 'PropertyFilterSpec' }, - propSet: [propertySpec], - objectSet: [objectSpec], - } - // TODO: research why it fails if propSet is declared after objectSet - - VItest.runCommand('RetrievePropertiesEx', { - _this: propertyCollector, - specSet: [propertyFilterSpec], - options: { attributes: { type: 'RetrieveOptions' } }, - }) - .once('result', function (result, raw) { - assert.notEqual(result.returnval.objects, null) - if (Array.isArray(result.returnval.objects)) { - assert.strictEqual(result.returnval.objects[0].obj.attributes.type, 'VirtualMachine') - } else { - assert.strictEqual(result.returnval.objects.obj.attributes.type, 'VirtualMachine') - } - done() - }) - .once('error', function (err) { - console.error('\n\nlast request : ' + VItest.client.lastRequest, err) - }) - }) - }) -}) diff --git a/@xen-orchestra/qa-test/.gitignore b/@xen-orchestra/qa-test/.gitignore deleted file mode 100644 index 11bb9fcc73..0000000000 --- a/@xen-orchestra/qa-test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/load-test-results/ diff --git a/@xen-orchestra/qa-test/utils/esxiMigrationUtils.js b/@xen-orchestra/qa-test/utils/esxiMigrationUtils.js index 3d7831ee4e..70ab5ba62a 100644 --- a/@xen-orchestra/qa-test/utils/esxiMigrationUtils.js +++ b/@xen-orchestra/qa-test/utils/esxiMigrationUtils.js @@ -237,8 +237,8 @@ export async function waitForSourceSnapshot(esxi, vmId, timeout = 120_000) { * snapshot. * * The snapshots are removed while the VM is halted so the consolidation does not compete with - * a live guest. Note `#waitForTaskEnd` gives a task 60 s, so a very large delta to consolidate - * would time out. + * a live guest. `removeAllSnapshots` gives the task 6 h, which a delta of any realistic size + * consolidates within. * * @param {Esxi} esxi - Connected ESXi client * @param {string} vmId - ESXi VM id diff --git a/@xen-orchestra/vmware-explorer/.USAGE.md b/@xen-orchestra/vmware-explorer/.USAGE.md new file mode 100644 index 0000000000..b4e016db6e --- /dev/null +++ b/@xen-orchestra/vmware-explorer/.USAGE.md @@ -0,0 +1,39 @@ +Reads the inventory, the metadata and the disks of an ESXi host or a vCenter, over vim25 SOAP and +the `/folder` HTTP endpoint, and exports the content of a disk through an `nbdkit` server. + +### Errors + +Every error raised by this package carries a `code`, so that a caller can branch on a failure +without matching its message. There are three families. + +**Faults of the host** are passed through verbatim, as the vim25 fault type: `FileLocked`, +`InvalidPowerState`, `ManagedObjectNotFound`, `NotAuthenticated`, `InvalidDeviceSpec`… These are the +codes to branch on whenever the host itself refused an operation, and they are documented in the +[vim25 API reference](https://developer.broadcom.com/xapis/vsphere-web-services-api/latest/). A task +which failed without the host naming a fault type reports `TASK_FAILED` instead. + +**Failures of the transport** keep the code of the error underneath: `ECONNREFUSED`, `ECONNRESET`, +`EHOSTUNREACH`, `ENOTFOUND`, `ETIMEDOUT`, `ENOENT` when a binary is missing, and the +`UND_ERR_*` codes of [undici](https://github.com/nodejs/undici). + +**Failures of this package** use the codes below. + +| code | meaning | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `BAD_VIM25_SPEC` | A vim25 spec was built with an element the schema does not declare. This one is a bug of this package, not a problem of the host. | +| `BAD_VMX_PATH` | The `files.vmPathName` of a VM is not `[datastore] dir/vm.vmx`. Carries `vmId`. | +| `DATACENTER_NOT_FOUND` | No datacenter holds the named datastore, and a file cannot be downloaded without one. Carries `dataStore` and the `dataStores` which are known. | +| `DATASTORE_NOT_FOUND` | An absolute disk reference of a vmx or a vmsd is on no known datastore. Carries `filePath` and `dataStoreUrls`. | +| `ESXI_SESSION_EXPIRED` | The session of the HTTP endpoint was refused (401 or 403), which happens on an import lasting hours. Retried after authenticating again. | +| `NBDKIT_EXITED` | The `nbdkit` server exited before it started listening, e.g. on a wrong thumbprint or a missing vddk library. Its logs are named in the message. | +| `NBDKIT_NOT_LISTENING` | The `nbdkit` server is running, but never started listening on its port. | +| `NO_DATA_MAP` | The allocated ranges of a disk could not be read: an unreadable vmdk descriptor, or a delta in a format which is not supported. The disk has to be transferred in full instead of as a delta. | +| `NO_PROPERTY` | The host returned no value for the property which was read, e.g. because the object is gone. | +| `NO_TASK` | A `*_Task` method answered without a task to wait for. | +| `RANGE_IGNORED` | The host answered something else than the byte range which was requested. Reading it would use the wrong offset, or as much memory as the file is big. | +| `TASK_FAILED` | A task ended in error, and the host named no fault type for it. | +| `TASK_TIMEOUT` | A task did not complete within its deadline. It keeps running on the host, so it is **not** retried: starting a second one would not make the first go away. | + +`ESXI_SESSION_EXPIRED` and the transport failures are transient, and are retried inside the package +— while downloading from a datastore, and while polling a task. Every other code is reported to the +caller on the first occurrence. diff --git a/@xen-orchestra/vmware-explorer/DatastoreSoapEsxi.mjs b/@xen-orchestra/vmware-explorer/DatastoreSoapEsxi.mjs deleted file mode 100644 index c2cd45b045..0000000000 --- a/@xen-orchestra/vmware-explorer/DatastoreSoapEsxi.mjs +++ /dev/null @@ -1,23 +0,0 @@ -import { Datastore } from './_Datastore.mjs' -import { Readable } from 'node:stream' - -export class DatastoreSoapEsxi extends Datastore { - #esxi - constructor(datastoreName, { esxi, ...otherOptions } = {}) { - super(datastoreName, otherOptions) - this.#esxi = esxi - } - async getStream(path, start, end) { - const res = await this.#esxi.download(this.datastoreName, path, start || end ? `${start}-${end}` : undefined) - return Readable.from(res.body, { objectMode: false, highWaterMark: 2 * 1024 * 1024 }) - } - async getBuffer(path, start, end) { - const res = await this.#esxi.download(this.datastoreName, path, start || end ? `${start}-${end - 1}` : undefined) - const bytes = await res.bytes() - return Buffer.from(bytes) - } - async getSize(path) { - const res = await this.#esxi.download(this.datastoreName, path) - return Number(res.headers.get('content-length')) - } -} diff --git a/@xen-orchestra/vmware-explorer/DatastoreXoRemote.mjs b/@xen-orchestra/vmware-explorer/DatastoreXoRemote.mjs deleted file mode 100644 index 481cce8105..0000000000 --- a/@xen-orchestra/vmware-explorer/DatastoreXoRemote.mjs +++ /dev/null @@ -1,22 +0,0 @@ -import assert from 'node:assert' -import { Datastore } from './_Datastore.mjs' - -export const DatastoreXoRemote = class DatastoreXoRemote extends Datastore { - #handler - constructor(datastoreName, { handler, ...otherOptions }) { - super(datastoreName, otherOptions) - this.#handler = handler - } - async getStream(path, start, end) { - return this.#handler.createReadStream(path, { start, end }) - } - async getBuffer(path, start, end) { - const buffer = Buffer.alloc(end - start) - const { bytesRead } = await this.#handler.read(path, buffer, start) - assert.strictEqual(bytesRead, end - start) - return buffer - } - async getSize(path) { - return this.#handler.getSize(path) - } -} diff --git a/@xen-orchestra/vmware-explorer/README.md b/@xen-orchestra/vmware-explorer/README.md new file mode 100644 index 0000000000..cc786a8267 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/README.md @@ -0,0 +1,72 @@ + + +# @xen-orchestra/vmware-explorer + +[![Package Version](https://badgen.net/npm/v/@xen-orchestra/vmware-explorer)](https://npmjs.org/package/@xen-orchestra/vmware-explorer) ![License](https://badgen.net/npm/license/@xen-orchestra/vmware-explorer) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/vmware-explorer)](https://bundlephobia.com/result?p=@xen-orchestra/vmware-explorer) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/vmware-explorer)](https://npmjs.org/package/@xen-orchestra/vmware-explorer) + +> Reads the VMs and the disks of an ESXi host or a vCenter + +## Install + +Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/vmware-explorer): + +```sh +npm install --save @xen-orchestra/vmware-explorer +``` + +## Usage + +Reads the inventory, the metadata and the disks of an ESXi host or a vCenter, over vim25 SOAP and +the `/folder` HTTP endpoint, and exports the content of a disk through an `nbdkit` server. + +### Errors + +Every error raised by this package carries a `code`, so that a caller can branch on a failure +without matching its message. There are three families. + +**Faults of the host** are passed through verbatim, as the vim25 fault type: `FileLocked`, +`InvalidPowerState`, `ManagedObjectNotFound`, `NotAuthenticated`, `InvalidDeviceSpec`… These are the +codes to branch on whenever the host itself refused an operation, and they are documented in the +[vim25 API reference](https://developer.broadcom.com/xapis/vsphere-web-services-api/latest/). A task +which failed without the host naming a fault type reports `TASK_FAILED` instead. + +**Failures of the transport** keep the code of the error underneath: `ECONNREFUSED`, `ECONNRESET`, +`EHOSTUNREACH`, `ENOTFOUND`, `ETIMEDOUT`, `ENOENT` when a binary is missing, and the +`UND_ERR_*` codes of [undici](https://github.com/nodejs/undici). + +**Failures of this package** use the codes below. + +| code | meaning | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `BAD_VIM25_SPEC` | A vim25 spec was built with an element the schema does not declare. This one is a bug of this package, not a problem of the host. | +| `BAD_VMX_PATH` | The `files.vmPathName` of a VM is not `[datastore] dir/vm.vmx`. Carries `vmId`. | +| `DATACENTER_NOT_FOUND` | No datacenter holds the named datastore, and a file cannot be downloaded without one. Carries `dataStore` and the `dataStores` which are known. | +| `DATASTORE_NOT_FOUND` | An absolute disk reference of a vmx or a vmsd is on no known datastore. Carries `filePath` and `dataStoreUrls`. | +| `ESXI_SESSION_EXPIRED` | The session of the HTTP endpoint was refused (401 or 403), which happens on an import lasting hours. Retried after authenticating again. | +| `NBDKIT_EXITED` | The `nbdkit` server exited before it started listening, e.g. on a wrong thumbprint or a missing vddk library. Its logs are named in the message. | +| `NBDKIT_NOT_LISTENING` | The `nbdkit` server is running, but never started listening on its port. | +| `NO_DATA_MAP` | The allocated ranges of a disk could not be read: an unreadable vmdk descriptor, or a delta in a format which is not supported. The disk has to be transferred in full instead of as a delta. | +| `NO_PROPERTY` | The host returned no value for the property which was read, e.g. because the object is gone. | +| `NO_TASK` | A `*_Task` method answered without a task to wait for. | +| `RANGE_IGNORED` | The host answered something else than the byte range which was requested. Reading it would use the wrong offset, or as much memory as the file is big. | +| `TASK_FAILED` | A task ended in error, and the host named no fault type for it. | +| `TASK_TIMEOUT` | A task did not complete within its deadline. It keeps running on the host, so it is **not** retried: starting a second one would not make the first go away. | + +`ESXI_SESSION_EXPIRED` and the transport failures are transient, and are retried inside the package +— while downloading from a datastore, and while polling a task. Every other code is reported to the +caller on the first occurrence. + +## Contributions + +Contributions are _very_ welcomed, either on the documentation or on +the code. + +You may: + +- report any [issue](https://github.com/vatesfr/xen-orchestra/issues) + you've encountered; +- fork and create a pull request. + +## License + +[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr) diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs deleted file mode 100644 index c855d31baa..0000000000 --- a/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs +++ /dev/null @@ -1,159 +0,0 @@ -import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' -import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' -import { FOOTER_SIZE } from 'vhd-lib/_constants.js' -import { notEqual, strictEqual } from 'node:assert' -import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' -import { VhdAbstract } from 'vhd-lib' -import { openDatastore } from './_openDatastore.mjs' - -export default class VhdEsxiCowd extends VhdAbstract { - #datastore - #parentVhd - #path - #lookMissingBlockInParent - - #header - #footer - - #grainDirectory - - static async open(datastoreName, path, parentVhd, opts) { - const datastore = openDatastore(datastoreName, opts) - const vhd = new VhdEsxiCowd(datastore, path, parentVhd, opts) - await vhd.readHeaderAndFooter() - return { value: vhd, dispose: () => {} } - } - constructor(datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) { - super() - this.#path = path - this.#datastore = datastore - this.#parentVhd = parentVhd - this.#lookMissingBlockInParent = lookMissingBlockInParent - } - - get header() { - return this.#header - } - - get footer() { - return this.#footer - } - - containsBlock(blockId) { - notEqual(this.#grainDirectory, undefined, "bat must be loaded to use contain blocks'") - // only check if a grain table exist for one of the sector of the block - // the great news is that a grain size has 4096 entries of 512B = 2M - // and a vhd block is also 2M - // so we only need to check if a grain table exists (it's not created without data) - - // depending on the parameters we also look into the parent data - return ( - this.#grainDirectory.readUInt32LE(blockId * 4) !== 0 || - (this.#lookMissingBlockInParent && this.#parentVhd.containsBlock(blockId)) - ) - } - - async #read(start, length) { - return this.#datastore.getBuffer(this.#path, start, start + length) - } - - async readHeaderAndFooter() { - const buffer = await this.#read(0, 2048) - - strictEqual(buffer.slice(0, 4).toString('ascii'), 'COWD') - strictEqual(buffer.readUInt32LE(4), 1) // version - strictEqual(buffer.readUInt32LE(8), 3) // flags - const numSectors = buffer.readUInt32LE(12) - const grainSize = buffer.readUInt32LE(16) - strictEqual(grainSize, 1) // 1 grain should be 1 sector long - strictEqual(buffer.readUInt32LE(20), 4) // grain directory position in sectors - - const size = numSectors * 512 - - this.#header = unpackHeader(createHeader(Math.ceil(size / (2 * 1024 * 1024) /* vhd block size */))) - const geometry = _computeGeometryForSize(size) - this.#footer = unpackFooter( - createFooter(size, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, this.#parentVhd.footer.diskType) - ) - } - - async readBlockAllocationTable() { - // a grain directory entry contains the address of a grain table - // a grain table can address at most 4096 grain of 512 Bytes of data - const nbBlocks = this.header.maxTableEntries - this.#grainDirectory = await this.#read(2048 /* header length */, nbBlocks * 4) - } - - // we're lucky : a grain address can address exactly a full block - async readBlock(blockId) { - notEqual(this.#grainDirectory, undefined, 'grainDirectory is not loaded') - const sectorOffset = this.#grainDirectory.readUInt32LE(blockId * 4) - - const buffer = (await this.#parentVhd.readBlock(blockId)).buffer - - if (sectorOffset === 0) { - strictEqual(this.#lookMissingBlockInParent, true, "shouldn't have empty block in a delta alone") - return { - id: blockId, - bitmap: buffer.slice(0, 512), - data: buffer.slice(512), - buffer, - } - } - const offset = sectorOffset * 512 - - const graintable = await this.#read(offset, 4096 * 4 /* grain table length */) - - strictEqual(graintable.length, 4096 * 4) - // we have no guaranty that data are ordered or contiguous - // let's construct ranges to limit the number of queries - let rangeStart, offsetStart, offsetEnd - - const changeRange = async (index, offset) => { - if (offsetStart !== undefined) { - // if there was already a branch - if (offset === offsetEnd) { - offsetEnd++ - return - } - const grains = await this.#read(offsetStart * 512, (offsetEnd - offsetStart) * 512) - grains.copy(buffer, (rangeStart + 1) /* block bitmap */ * 512) - } - // start a new range - if (offset) { - // we're at the beginning of a range present in the file - rangeStart = index - offsetStart = offset - offsetEnd = offset + 1 - } else { - // we're at the beginning of a range from the parent or empty - rangeStart = undefined - offsetStart = undefined - offsetEnd = undefined - } - } - - for (let i = 0; i < graintable.length / 4; i++) { - const grainOffset = graintable.readUInt32LE(i * 4) - if (grainOffset === 0) { - // the content from parent : it is already in buffer - await changeRange() - } else if (grainOffset === 1) { - await changeRange() - // this is a emptied grain, no data, don't look into parent - buffer.fill(0, (i + 1) /* block bitmap */ * 512) - } else if (grainOffset > 1) { - // non empty grain, read from file - await changeRange(i, grainOffset) - } - } - // close last range - await changeRange() - return { - id: blockId, - bitmap: buffer.slice(0, 512), - data: buffer.slice(512), - buffer, - } - } -} diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs deleted file mode 100644 index d359387b62..0000000000 --- a/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs +++ /dev/null @@ -1,215 +0,0 @@ -import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' -import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' -import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js' -import { readChunkStrict, skipStrict } from '@vates/read-chunk' -import { Task } from '@vates/task' -import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' -import { VhdAbstract } from 'vhd-lib' -import assert from 'node:assert' -import { openDatastore } from './_openDatastore.mjs' - -/* eslint-disable no-console */ - -// create a thin VHD from a raw disk -const VHD_BLOCK_LENGTH = 2 * 1024 * 1024 -export default class VhdEsxiRaw extends VhdAbstract { - #datastore - #path - #thin - - #bat - #header - #footer - - #streamOffset = 0 - #stream - #reading = false - - static async open(datastoreName, path, opts) { - const datastore = openDatastore(datastoreName, opts) - const vhd = new VhdEsxiRaw(datastore, path, opts) - await vhd.readHeaderAndFooter() - return { - value: vhd, - dispose: () => vhd.dispose(), - } - } - - get header() { - return this.#header - } - - get footer() { - return this.#footer - } - - constructor(datastore, path, { thin = true } = {}) { - super() - this.#path = path - this.#datastore = datastore - this.#thin = thin - } - - async readHeaderAndFooter() { - const length = await this.#datastore.getSize(this.#path) - - this.#header = unpackHeader(createHeader(length / VHD_BLOCK_LENGTH)) - const geometry = _computeGeometryForSize(length) - - this.#footer = unpackFooter( - // length can be smaller than disk capacity due to alignment to head/cylinder/sector - createFooter(length, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC) - ) - } - - containsBlock(blockId) { - if (!this.#thin) { - return true - } - assert.notEqual(this.#bat, undefined, 'bat is not loaded') - return this.#bat.has(blockId) - } - - async #readChunk(start, length) { - if (this.#reading) { - throw new Error('reading must be done sequentially') - } - try { - this.#reading = true - if (this.#stream !== undefined) { - // stream is too far ahead or to far behind - if (this.#streamOffset > start || this.#streamOffset + VHD_BLOCK_LENGTH < start) { - this.#stream.destroy() - this.#stream = undefined - this.#streamOffset = 0 - } - } - // no stream - if (this.#stream === undefined) { - const end = this.footer.currentSize - 1 - this.#stream = await this.#datastore.getStream(this.#path, start, end) - this.#streamOffset = start - } - - // stream a little behind - if (this.#streamOffset < start) { - await skipStrict(this.#stream, start - this.#streamOffset) - this.#streamOffset = start - } - - // really read data - this.#streamOffset += length - const data = await readChunkStrict(this.#stream, length) - return data - } catch (error) { - error.start = start - error.length = length - error.streamLength = this.footer.currentSize - this.#stream?.destroy() - this.#stream = undefined - this.#streamOffset = 0 - throw error - } finally { - this.#reading = false - } - } - - async #readBlock(blockId) { - const start = blockId * VHD_BLOCK_LENGTH - let length = VHD_BLOCK_LENGTH - let partial = false - if (start + length > this.footer.currentSize) { - length = this.footer.currentSize - start - partial = true - } - - let data = await this.#readChunk(start, length) - - if (partial) { - data = Buffer.concat([data, Buffer.alloc(VHD_BLOCK_LENGTH - data.length)]) - } - const bitmap = Buffer.alloc(512, 255) - return { - id: blockId, - bitmap, - data, - buffer: Buffer.concat([bitmap, data]), - } - } - - async readBlock(blockId) { - let tries = 5 - let lastError - while (tries > 0) { - try { - const res = await this.#readBlock(blockId) - return res - } catch (error) { - lastError = error - lastError.blockId = blockId - console.warn('got error , will retry in 2seconds', lastError) - } - await new Promise(resolve => setTimeout(() => resolve(), 2000)) - tries-- - } - - throw lastError - } - - // this will read all the disk once to check which block contains data, it can take a long time to execute depending on the network speed - async readBlockAllocationTable() { - if (!this.#thin) { - // fast path : is we do not use thin mode, the BAT is full - return - } - const empty = Buffer.alloc(VHD_BLOCK_LENGTH, 0) - let pos = 0 - this.#bat = new Set() - let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, this.footer.currentSize) - Task.set('total', this.footer.currentSize / VHD_BLOCK_LENGTH) - const progress = setInterval(() => { - Task.set('progress', Math.round((pos * 100) / this.footer.currentSize)) - console.log('reading blocks', pos / VHD_BLOCK_LENGTH, '/', this.footer.currentSize / VHD_BLOCK_LENGTH) - }, 30 * 1000) - - while (nextChunkLength > 0) { - try { - const chunk = await this.#readChunk(pos, nextChunkLength) - let isEmpty - if (nextChunkLength === VHD_BLOCK_LENGTH) { - isEmpty = empty.equals(chunk) - } else { - // last block can be smaller - isEmpty = Buffer.alloc(nextChunkLength, 0).equals(chunk) - } - if (!isEmpty) { - this.#bat.add(pos / VHD_BLOCK_LENGTH) - } - pos += VHD_BLOCK_LENGTH - nextChunkLength = Math.min(VHD_BLOCK_LENGTH, this.footer.currentSize - pos) - } catch (error) { - clearInterval(progress) - throw error - } - } - console.log( - 'BAT reading done, remaining ', - this.#bat.size, - '/', - Math.ceil(this.footer.currentSize / VHD_BLOCK_LENGTH) - ) - clearInterval(progress) - } - - rawContent() { - return this.#datastore.getStream(this.#path).then(stream => { - stream.length = this.footer.currentSize - return stream - }) - } - async dispose() { - await this.#stream?.destroy() - } -} - -/* eslint-enable no-console */ diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs deleted file mode 100644 index 663fcc584c..0000000000 --- a/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs +++ /dev/null @@ -1,312 +0,0 @@ -import { readChunkStrict, skipStrict } from '@vates/read-chunk' -import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' -import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' -import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js' -import { notEqual, strictEqual } from 'node:assert' -import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' -import { VhdAbstract } from 'vhd-lib' -import { openDatastore } from './_openDatastore.mjs' - -// one big difference with the other versions of VMDK is that the grain tables are actually sparse, they are pre-allocated but not used in grain order, -// so we have to read the grain directory to know where to find the grain tables - -const SE_SPARSE_DIR_NON_ALLOCATED = 0 -const SE_SPARSE_DIR_ALLOCATED = 1 - -const SE_SPARSE_GRAIN_NON_ALLOCATED = 0 // check in parent -const SE_SPARSE_GRAIN_UNMAPPED = 1 // grain has been unmapped, but index of previous grain still readable for reclamation -const SE_SPARSE_GRAIN_ZERO = 2 -const SE_SPARSE_GRAIN_ALLOCATED = 3 - -const VHD_BLOCK_SIZE_BYTES = 2 * 1024 * 1024 -const GRAIN_SIZE_BYTES = 4 * 1024 -const GRAIN_TABLE_COUNT = 4 * 1024 - -const ones = n => (1n << BigInt(n)) - 1n - -function asNumber(n) { - if (n > Number.MAX_SAFE_INTEGER) - throw new Error(`can't handle ${n} ${Number.MAX_SAFE_INTEGER} ${n & 0x00000000ffffffffn}`) - return Number(n) -} - -const readInt64 = (buffer, index) => asNumber(buffer.readBigInt64LE(index * 8)) - -/** - * @returns {{topNibble: number, low60: bigint}} topNibble is the first 4 bits of the 64 bits entry, indexPart is the remaining 60 bits - */ -function readTaggedEntry(buffer, index) { - const entry = buffer.readBigInt64LE(index * 8) - return { topNibble: Number(entry >> 60n), low60: entry & ones(60) } -} - -function readSeSparseDir(buffer, index) { - const { topNibble, low60 } = readTaggedEntry(buffer, index) - return { type: topNibble, tableIndex: asNumber(low60) } -} - -function readSeSparseTable(buffer, index) { - const { topNibble, low60 } = readTaggedEntry(buffer, index) - // https://lists.gnu.org/archive/html/qemu-block/2019-06/msg00934.html - const topIndexPart = low60 >> 48n // bring the top 12 bits down - const bottomIndexPart = (low60 & ones(48)) << 12n // bring the bottom 48 bits up - return { type: topNibble, grainIndex: asNumber(bottomIndexPart | topIndexPart) } -} - -export default class VhdEsxiSeSparse extends VhdAbstract { - #datastore - #parentVhd - #path - #lookMissingBlockInParent - - #header - #footer - - #grainIndex // Map blockId => [] - - #grainDirOffsetBytes - #grainDirSizeBytes - #grainTableOffsetBytes - #grainOffsetBytes - - #reading = false - #stream - #streamOffset = 0 - - static async open(datastoreName, path, parentVhd, opts) { - const datastore = openDatastore(datastoreName, opts) - const vhd = new VhdEsxiSeSparse(datastore, path, parentVhd, opts) - await vhd.readHeaderAndFooter() - return { - value: vhd, - dispose: () => vhd.dispose(), - } - } - - get path() { - return this.#path - } - constructor(datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) { - super() - this.#path = path - this.#datastore = datastore - this.#parentVhd = parentVhd - this.#lookMissingBlockInParent = lookMissingBlockInParent - } - - get header() { - return this.#header - } - - get footer() { - return this.#footer - } - - containsBlock(blockId) { - notEqual(this.#grainIndex, undefined, "bat must be loaded to use contain blocks'") - return ( - this.#grainIndex.get(blockId) !== undefined || - (this.#lookMissingBlockInParent && this.#parentVhd.containsBlock(blockId)) - ) - } - - // since most of the data are written sequentially we always open a stream from start to the end of the file - // If we have to rewind it, we destroy the stream and recreate with the right "start" - // We also recreate the stream if there is too much distance between current position and the wanted position - - async #read(start, length) { - if (!this.#footer) { - // we need to be able before the footer is loaded, to read the header and footer - return this.#datastore.getBuffer(this.#path, start, start + length) - } - if (this.#reading) { - throw new Error('reading must be done sequentially') - } - try { - const MAX_SKIPPABLE_LENGTH = 2 * 1024 * 1024 - this.#reading = true - if (this.#stream !== undefined) { - // stream is already ahead or to far behind - if (this.#streamOffset > start || this.#streamOffset + MAX_SKIPPABLE_LENGTH < start) { - this.#stream.destroy() - this.#stream = undefined - this.#streamOffset = 0 - } - } - // no stream - if (this.#stream === undefined) { - const end = this.footer.currentSize - this.#stream = await this.#datastore.getStream(this.#path, start, end) - this.#streamOffset = start - } - - // stream a little behind - if (this.#streamOffset < start) { - await skipStrict(this.#stream, start - this.#streamOffset) - this.#streamOffset = start - } - - // really read data - this.#streamOffset += length - const data = await readChunkStrict(this.#stream, length) - return data - } catch (error) { - error.start = start - error.length = length - error.streamLength = this.footer.currentSize - this.#stream?.destroy() - this.#stream = undefined - this.#streamOffset = 0 - throw error - } finally { - this.#reading = false - } - } - - async readHeaderAndFooter() { - const vmdkHeaderBuffer = await this.#read(0, 2048) - - strictEqual(vmdkHeaderBuffer.readBigInt64LE(0), 0xcafebaben) - strictEqual(readInt64(vmdkHeaderBuffer, 1), 0x200000001) // version 2.1 - - this.#grainDirOffsetBytes = readInt64(vmdkHeaderBuffer, 16) * 512 - // console.log('grainDirOffsetBytes', this.#grainDirOffsetBytes) - this.#grainDirSizeBytes = readInt64(vmdkHeaderBuffer, 17) * 512 - // console.log('grainDirSizeBytes', this.#grainDirSizeBytes) - - const grainSizeSectors = readInt64(vmdkHeaderBuffer, 3) - const grainSizeBytes = grainSizeSectors * 512 // 8 sectors = 4KB default - strictEqual(grainSizeBytes, GRAIN_SIZE_BYTES) // we only support default grain size - - this.#grainTableOffsetBytes = readInt64(vmdkHeaderBuffer, 18) * 512 - // console.log('grainTableOffsetBytes', this.#grainTableOffsetBytes) - - const grainTableCount = (readInt64(vmdkHeaderBuffer, 4) * 512) / 8 // count is the number of 64b entries in each tables - // console.log('grainTableCount', grainTableCount) - strictEqual(grainTableCount, GRAIN_TABLE_COUNT) // we only support tables of 4096 entries (default) - - this.#grainOffsetBytes = readInt64(vmdkHeaderBuffer, 24) * 512 - // console.log('grainOffsetBytes', this.#grainOffsetBytes) - - const sizeBytes = readInt64(vmdkHeaderBuffer, 2) * 512 - // console.log('sizeBytes', sizeBytes) - - const nbBlocks = Math.ceil(sizeBytes / VHD_BLOCK_SIZE_BYTES) - this.#header = unpackHeader(createHeader(nbBlocks)) - const geometry = _computeGeometryForSize(sizeBytes) - this.#footer = unpackFooter( - createFooter(sizeBytes, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC) - ) - } - - async readBlockAllocationTable() { - this.#grainIndex = new Map() - - const tableSizeBytes = GRAIN_TABLE_COUNT * 8 - const grainDirBuffer = await this.#read(this.#grainDirOffsetBytes, this.#grainDirSizeBytes) - // read the grain dir ( first level ) - for (let grainDirIndex = 0; grainDirIndex < grainDirBuffer.length / 8; grainDirIndex++) { - const { type: grainDirType, tableIndex } = readSeSparseDir(grainDirBuffer, grainDirIndex) - if (grainDirType === SE_SPARSE_DIR_NON_ALLOCATED) { - // no grain table allocated at all in this grain dir - continue - } - strictEqual(grainDirType, SE_SPARSE_DIR_ALLOCATED) - // read the corresponding grain table ( second level ) - const grainTableBuffer = await this.#read( - this.#grainTableOffsetBytes + tableIndex * tableSizeBytes, - tableSizeBytes - ) - // offset in bytes if >0, grainType if <=0 - let grainOffsets = [] - let blockId = grainDirIndex * 8 - - const addGrain = val => { - grainOffsets.push(val) - // 4096 block of 4Kb per dir entry =>16MB/grain dir - // 1 block = 2MB - // 512 grain => 1 block - // 8 block per dir entry - if (grainOffsets.length === 512) { - this.#grainIndex.set(blockId, grainOffsets) - grainOffsets = [] - blockId++ - } - } - - for (let grainTableIndex = 0; grainTableIndex < grainTableBuffer.length / 8; grainTableIndex++) { - const { type: grainType, grainIndex } = readSeSparseTable(grainTableBuffer, grainTableIndex) - if (grainType === SE_SPARSE_GRAIN_ALLOCATED) { - // this is ok in 32 bits int with VMDK smaller than 2TB - const offsetByte = grainIndex * GRAIN_SIZE_BYTES + this.#grainOffsetBytes - addGrain(offsetByte) - } else { - // multiply by -1 to differentiate type and offset - // no offset can be zero - addGrain(-grainType) - } - } - strictEqual(grainOffsets.length, 0) - } - } - - async readBlock(blockId) { - let changed = false - const grainOffsets = this.#grainIndex.get(blockId) // may be undefined if the child contains block and lookMissingBlockInParent=true - // negative value indicate that it's not an offset - // SE_SPARSE_GRAIN_NON_ALLOCATED means we have to look into the parent data - const isLocallyFull = - grainOffsets !== undefined && !grainOffsets.some(value => value === -SE_SPARSE_GRAIN_NON_ALLOCATED) - - let parentBuffer, parentBlock - // don't read from parent is current block is already completely described - if (isLocallyFull) { - parentBuffer = Buffer.alloc(512 /* bitmap */ + 2 * 1024 * 1024 /* data */, 0) - parentBuffer.fill(255, 0, 512) // bitmap is full of bit 1 - } else { - parentBlock = await this.#parentVhd.readBlock(blockId) - parentBuffer = parentBlock.buffer - } - const EMPTY_GRAIN = Buffer.alloc(GRAIN_SIZE_BYTES, 0) - for (const index in grainOffsets) { - const value = grainOffsets[index] - let data - if (value > 0) { - // it's the offset in byte of a grain type SE_SPARSE_GRAIN_ALLOCATED - // @todo this part can be quite slow when grain are not sorted - data = await this.#read(value, GRAIN_SIZE_BYTES) - } else { - // back to the real grain type - const type = value * -1 - switch (type) { - case SE_SPARSE_GRAIN_ZERO: - case SE_SPARSE_GRAIN_UNMAPPED: - data = EMPTY_GRAIN - break - case SE_SPARSE_GRAIN_NON_ALLOCATED: - /* from parent */ - break - default: - throw new Error(`can't handle grain type ${type}`) - } - } - if (data) { - changed = true - data.copy(parentBuffer, index * GRAIN_SIZE_BYTES + 512 /* block bitmap */) - } - } - // no need to copy if data all come from parent - return changed || !parentBlock - ? { - id: blockId, - bitmap: parentBuffer.slice(0, 512), - data: parentBuffer.slice(512), - buffer: parentBuffer, - } - : parentBlock - } - - async dispose() { - await this.#stream?.destroy() - } -} diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiStreamOptimized.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiStreamOptimized.mjs deleted file mode 100644 index 2a730f82f1..0000000000 --- a/@xen-orchestra/vmware-explorer/VhdEsxiStreamOptimized.mjs +++ /dev/null @@ -1,152 +0,0 @@ -import { VhdAbstract } from 'vhd-lib' -import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' -import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' -import { - DEFAULT_BLOCK_SIZE as DEFAULT_VHD_BLOCK_SIZE, - DISK_TYPES, - FOOTER_SIZE, - SECTOR_SIZE, -} from 'vhd-lib/_constants.js' -import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' -import { unpackHeader as unpackVmdkHeader } from 'xo-vmdk-to-vhd/dist/definitions.js' - -import zlib from 'zlib' -import assert from 'node:assert' - -export default class VhdEsxiStreamOptimized extends VhdAbstract { - #handler - #vmdkHeader - #l1table - #l2tables - #path - - get header() { - // a grain directory entry contains the address of a grain table - // a grain table can addresses at most 4096 grain of 512 Bytes of data - return unpackHeader( - createHeader(Math.ceil((this.#vmdkHeader.capacitySectors * SECTOR_SIZE) / DEFAULT_VHD_BLOCK_SIZE)) - ) - } - - get footer() { - const size = this.#vmdkHeader.capacitySectors * SECTOR_SIZE - const geometry = _computeGeometryForSize(size) - return unpackFooter(createFooter(size, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)) - } - - constructor(handler, path) { - super() - this.#handler = handler - this.#path = path - } - async readHeaderAndFooter() { - // complete header is at the end of the file - // the header at the beginning don't have the L1 table position - const size = await this.#handler.getSize(this.#path) - const headerBuffer = await this.#read(size - 1024, 1024) - this.#vmdkHeader = unpackVmdkHeader(headerBuffer) - } - async readBlockAllocationTable() { - const l1entries = Math.floor( - (this.#vmdkHeader.capacitySectors + this.#vmdkHeader.l1EntrySectors - 1) / this.#vmdkHeader.l1EntrySectors - ) - const buffer = await this.#read(this.#vmdkHeader.grainDirectoryOffsetSectors * SECTOR_SIZE, l1entries * 4) - this.#l1table = [] - for (let grainTableIndex = 0; grainTableIndex < l1entries; grainTableIndex++) { - this.#l1table.push(buffer.readUInt32LE(grainTableIndex * 4)) - } - // also read all the l2 table since we must be able to check contains synchronously - this.#l2tables = [] - for (let grainTableIndex = 0; grainTableIndex < l1entries; grainTableIndex++) { - this.#l2tables[grainTableIndex] = await this.#getGrainTable(grainTableIndex) - } - } - async #read(start, length) { - const buffer = Buffer.alloc(length, 0) - await this.#handler.read(this.#path, buffer, start) - return buffer - } - async #getGrainTable(grainTableIndex) { - const l1Entry = this.#l1table[grainTableIndex] - if (l1Entry === 0) { - return - } - const l2ByteSize = this.#vmdkHeader.numGTEsPerGT * 4 - const buffer = await this.#read(l1Entry * SECTOR_SIZE, l2ByteSize) - const table = [] - for (let i = 0; i < this.#vmdkHeader.numGTEsPerGT; i++) { - table.push(buffer.readUInt32LE(i * 4)) - } - return table - } - - async #getGrainData(grainTableIndex, grainEntryIndex) { - const grainTable = this.#l2tables[grainTableIndex] - if (grainTable === undefined) { - return - } - const offset = grainTable[grainEntryIndex] - if (offset === 0) { - return - } - const dataSizeBuffer = await this.#read(offset * SECTOR_SIZE + 8, 4) - const dataSize = dataSizeBuffer.readUInt32LE(0) - const buffer = await this.#read(offset * SECTOR_SIZE + 12, dataSize) - const inflated = zlib.inflateSync(buffer) - assert(inflated.length, 64 * 1024) - return inflated - } - containsBlock(blockId) { - const nbBlocksPerGT = this.#getNumberOfBlockPerGrainTable() - - const grainTableIndex = Math.floor(blockId / nbBlocksPerGT) - const length = this.#getNumberOfGrainPerBlock() - const grainTableEntryIndexStart = (blockId % nbBlocksPerGT) * length - - const grainTable = this.#l2tables[grainTableIndex] - if (!grainTable) { - return false - } - - for (let i = 0; i < length; i++) { - const grainEntryIndex = grainTableEntryIndexStart + i - if (grainTable[grainEntryIndex] !== 0) { - return true - } - } - return false - } - - #getGrainSize() { - return this.#vmdkHeader.grainSizeSectors * SECTOR_SIZE - } - #getNumberOfBlockPerGrainTable() { - return (this.#vmdkHeader.numGTEsPerGT * this.#getGrainSize()) / DEFAULT_VHD_BLOCK_SIZE - } - #getNumberOfGrainPerBlock() { - return DEFAULT_VHD_BLOCK_SIZE / this.#getGrainSize() - } - async readBlock(blockId) { - // 1 grain = 128 sectors = 64KB - // 512 grain per grain table => 32MB per grain table, 16 vhd blocks - const nbBlocksPerGT = this.#getNumberOfBlockPerGrainTable() - const buffer = Buffer.alloc(512 /* bitmap */ + DEFAULT_VHD_BLOCK_SIZE /* data */, 0) - - const grainTableIndex = Math.floor(blockId / nbBlocksPerGT) - const length = this.#getNumberOfGrainPerBlock() - const grainTableEntryIndexStart = (blockId % nbBlocksPerGT) * length - for (let i = 0; i < length; i++) { - const grainEntryIndex = grainTableEntryIndexStart + i - const grain = await this.#getGrainData(grainTableIndex, grainEntryIndex) - if (grain !== undefined) { - grain.copy(buffer, i * grain.length + 512 /* bitmap */) - } - } - return { - id: blockId, - bitmap: buffer.slice(0, 512), - data: buffer.slice(512), - buffer, - } - } -} diff --git a/@xen-orchestra/vmware-explorer/_Datastore.mjs b/@xen-orchestra/vmware-explorer/_Datastore.mjs deleted file mode 100644 index 998f2ea3b6..0000000000 --- a/@xen-orchestra/vmware-explorer/_Datastore.mjs +++ /dev/null @@ -1,17 +0,0 @@ -export class Datastore { - #datastoreName - - get datastoreName() { - return this.#datastoreName - } - constructor(dataStoreName) { - this.#datastoreName = dataStoreName - } - async getStream(path, start, end) { - throw new Error('Not implemented') - } - async getBuffer(path, start, end) { - throw new Error('Not implemented') - } - async getSize(path) {} -} diff --git a/@xen-orchestra/vmware-explorer/_nbdkit.integ.mjs b/@xen-orchestra/vmware-explorer/_nbdkit.integ.mjs new file mode 100644 index 0000000000..9455b56016 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_nbdkit.integ.mjs @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict' +import { createServer } from 'node:net' +import { describe, it } from 'node:test' + +import { findFreePort, waitForPort } from './_nbdkit.mjs' + +// these bind a real port +describe('findFreePort', function () { + it('returns a port which can be bound', async function () { + const port = await findFreePort() + + assert.equal(typeof port, 'number') + const server = createServer() + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(port, '127.0.0.1', resolve) + }) + server.close() + }) +}) + +describe('waitForPort', function () { + it('resolves as soon as the port accepts a connection', async function () { + const port = await findFreePort() + const server = createServer() + // start listening later, as nbdkit does once its plugin is configured + const timer = setTimeout(() => server.listen(port, '127.0.0.1'), 30) + + try { + await waitForPort(port, { probeDelay: 5, timeout: 2e3 }) + } finally { + clearTimeout(timer) + server.close() + } + }) + + it('gives up with the reason of the last attempt', async function () { + const port = await findFreePort() + + await assert.rejects(waitForPort(port, { probeDelay: 1, timeout: 20 }), error => { + assert.match(error.message, /nothing is listening on 127\.0\.0\.1:/) + assert.equal(error.code, 'NBDKIT_NOT_LISTENING') + assert.equal(error.cause.code, 'ECONNREFUSED') + return true + }) + }) + + it('stops probing when aborted', async function () { + const port = await findFreePort() + + await assert.rejects(waitForPort(port, { probeDelay: 1, signal: AbortSignal.abort(), timeout: 1e3 }), { + name: 'AbortError', + }) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/_nbdkit.mjs b/@xen-orchestra/vmware-explorer/_nbdkit.mjs new file mode 100644 index 0000000000..f3524553ba --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_nbdkit.mjs @@ -0,0 +1,109 @@ +import { connect, createServer } from 'node:net' +import { setTimeout as delay } from 'node:timers/promises' + +/** + * Command line of an nbdkit server exporting one disk of a VM through the vddk plugin. + * + * @param {object} params + * @returns {Array} + */ +export function formatNbdkitArgs({ + compression, + diskPath, + host, + libdir, + passFile, + port, + singleLink, + threads, + thumbprint, + user, + vmId, +}) { + return [ + '-r', // readonly + '-v', + '-f', + '--exit-with-parent', // implies -f , ensure we don't leave orphans + `--threads=${threads}`, + `--port=${port}`, + 'vddk', // the vddk plugin + `compression=${compression}`, + `thumbprint=${thumbprint}`, + `server=${host}`, + `user=${user}`, + `password=+${passFile}`, + `libdir=${libdir}`, + `vm=moref=${vmId}`, + // `single-link` restricts the export to the top delta. An empty argument used to be passed + // when it is disabled: nbdkit tolerates it, but it sits in the slot of the magic positional + // parameter of the plugin, right before the disk to export + ...(singleLink ? ['single-link=true'] : []), + diskPath, + ] +} + +/** + * Reserves a port by letting the system pick a free one. + * + * The port is released before being handed out, so the caller must be able to cope with a + * concurrent process taking it in the meantime. + * + * @returns {Promise} + */ +export function findFreePort() { + return new Promise((resolve, reject) => { + const server = createServer() + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() + server.close(error => (error != null ? reject(error) : resolve(port))) + }) + }) +} + +const tryConnect = (port, host) => + new Promise((resolve, reject) => { + const socket = connect({ port, host }) + socket.once('connect', () => { + socket.destroy() + resolve() + }) + socket.once('error', error => { + socket.destroy() + reject(error) + }) + }) + +/** + * Waits until something listens on a port. + * + * nbdkit binds its port once its plugin is configured, so this is what tells that the server is + * usable — the previous implementation waited a fixed two seconds, which was both too long for a + * fast host and too short for a slow one. + * + * @param {number} port + * @param {object} [options] + * @param {string} [options.host] + * @param {number} [options.probeDelay] - in ms, between two attempts + * @param {AbortSignal} [options.signal] + * @param {number} [options.timeout] - in ms + * @returns {Promise} + */ +export async function waitForPort(port, { host = '127.0.0.1', probeDelay = 50, signal, timeout = 30e3 } = {}) { + const start = Date.now() + for (;;) { + signal?.throwIfAborted() + try { + return await tryConnect(port, host) + } catch (error) { + if (Date.now() - start >= timeout) { + const wrapped = new Error(`nothing is listening on ${host}:${port} after ${timeout}ms`) + wrapped.code = 'NBDKIT_NOT_LISTENING' + wrapped.cause = error + throw wrapped + } + } + await delay(probeDelay, undefined, { signal }) + } +} diff --git a/@xen-orchestra/vmware-explorer/_nbdkit.test.mjs b/@xen-orchestra/vmware-explorer/_nbdkit.test.mjs new file mode 100644 index 0000000000..09513df223 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_nbdkit.test.mjs @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { formatNbdkitArgs } from './_nbdkit.mjs' + +const PARAMS = { + compression: 'fastlz', + diskPath: '[ds main] a.vm/a.vm.vmdk', + host: 'esxi.test', + libdir: '/usr/local/lib/vddk/vmware-vix-disklib-distrib', + passFile: '/tmp/xo-server-42/params', + port: 11000, + singleLink: false, + threads: 1, + thumbprint: 'AA:BB:CC', + user: 'root', + vmId: 'vm-42', +} + +describe('formatNbdkitArgs', function () { + it('never passes an empty argument', function () { + // an empty argument sits in the slot of the magic positional parameter of the plugin + for (const singleLink of [false, true]) { + const args = formatNbdkitArgs({ ...PARAMS, singleLink }) + assert.equal( + args.every(argument => argument.length > 0), + true, + JSON.stringify(args) + ) + } + }) + + it('exports read only, exits with its parent, and reads the password from a file', function () { + const args = formatNbdkitArgs(PARAMS) + + assert.ok(args.includes('-r')) + assert.ok(args.includes('--exit-with-parent')) + assert.ok(args.includes(`password=+${PARAMS.passFile}`)) + // the password itself never appears in the command line, which is world readable + assert.equal( + args.some(argument => argument.includes('password=') && !argument.startsWith('password=+')), + false + ) + }) + + it('restricts the export to the top delta only when asked', function () { + assert.equal(formatNbdkitArgs(PARAMS).includes('single-link=true'), false) + assert.equal(formatNbdkitArgs({ ...PARAMS, singleLink: true }).includes('single-link=true'), true) + }) + + it('ends with the disk to export', function () { + const args = formatNbdkitArgs({ ...PARAMS, singleLink: true }) + + assert.equal(args[args.length - 1], PARAMS.diskPath) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/_openDatastore.mjs b/@xen-orchestra/vmware-explorer/_openDatastore.mjs deleted file mode 100644 index d8c98f080c..0000000000 --- a/@xen-orchestra/vmware-explorer/_openDatastore.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import { DatastoreSoapEsxi } from './DatastoreSoapEsxi.mjs' -import { DatastoreXoRemote } from './DatastoreXoRemote.mjs' -import { createLogger } from '@xen-orchestra/log' - -const { info } = createLogger('xo:vmware-explorer:openDatastore') - -export function openDatastore(dataStoreName, { esxi, dataStoreToHandlers = {}, ...otherOptions }) { - const handler = dataStoreToHandlers[dataStoreName] - if (handler === undefined) { - info(`use SOAP API to read datastore ${dataStoreName}`, { - dataStoreName, - dataStoreToHandlers: Object.keys(dataStoreToHandlers), - }) - return new DatastoreSoapEsxi(dataStoreName, { esxi, ...otherOptions }) - } - info(`use XO remote to read datastore ${dataStoreName}`, { - dataStoreName, - dataStoreToHandlers: Object.keys(dataStoreToHandlers), - }) - return new DatastoreXoRemote(dataStoreName, { handler, ...otherOptions }) -} diff --git a/@xen-orchestra/vmware-explorer/_paths.mjs b/@xen-orchestra/vmware-explorer/_paths.mjs new file mode 100644 index 0000000000..f045fe274a --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_paths.mjs @@ -0,0 +1,54 @@ +/** + * A datastore path is expressed in multiple ways depending on its source: + * - `ds:///vmfs/volumes//` in the `summary.url` reported by vCenter + * - `/vmfs/volumes/` in the `summary.url` reported by an ESXi host + * - `/vmfs/volumes///.vmdk` in the vmx of a VM ( typical of vSAN ) + * + * They are normalized before being compared. + * + * @param {string} path + * @returns {string} + */ +export function normalizeDatastorePath(path) { + return path.replace(/^ds:\/\//, '').replace(/\/+$/, '') +} + +/** + * Resolves the datastore and the path of a disk referenced by a vmx or a vmsd. + * + * The reference is either relative to the directory of the VM, or absolute in the file system of + * the host, in which case the datastore containing it has to be found. + * + * @param {object} params + * @param {Record} params.dataStores - datastore summaries, indexed by url + * @param {string} params.currentDataStore - name of the datastore holding the vmx + * @param {string} params.currentPath - directory of the vmx, in its datastore + * @param {string} params.filePath - reference to resolve + * @returns {{ dataStore: string, path: string }} + */ +export function resolveDiskLocation({ dataStores, currentDataStore, currentPath, filePath }) { + if (!/^(ds:\/\/)?\//.test(filePath)) { + return { dataStore: currentDataStore, path: `${currentPath}/${filePath}` } + } + + const normalizedFilePath = normalizeDatastorePath(filePath) + for (const [dataStoreUrl, dataStore] of Object.entries(dataStores)) { + const prefix = normalizeDatastorePath(dataStoreUrl) + if (prefix === '') { + // a datastore whose `summary.url` is empty or is only `ds:///` would match every absolute + // path, and the first one wins: the disk would be looked for on the wrong datastore + continue + } + if (normalizedFilePath.startsWith(prefix + '/')) { + return { dataStore: dataStore.name, path: normalizedFilePath.substring(prefix.length + 1) } + } + } + + // the disk used to be looked for on a datastore named `undefined`, and the failure surfaced much + // later as an assertion error + const error = new Error(`can't find the datastore containing the disk ${filePath}`) + error.code = 'DATASTORE_NOT_FOUND' + error.dataStoreUrls = Object.keys(dataStores) + error.filePath = filePath + throw error +} diff --git a/@xen-orchestra/vmware-explorer/_paths.test.mjs b/@xen-orchestra/vmware-explorer/_paths.test.mjs new file mode 100644 index 0000000000..a6d2b0b8a6 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_paths.test.mjs @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { normalizeDatastorePath, resolveDiskLocation } from './_paths.mjs' + +describe('normalizeDatastorePath', function () { + it('drops the ds:// scheme and the trailing slashes', function () { + assert.equal(normalizeDatastorePath('ds:///vmfs/volumes/uuid-1/'), '/vmfs/volumes/uuid-1') + assert.equal(normalizeDatastorePath('/vmfs/volumes/uuid-1'), '/vmfs/volumes/uuid-1') + assert.equal(normalizeDatastorePath('/vmfs/volumes/uuid-1///'), '/vmfs/volumes/uuid-1') + }) +}) + +describe('resolveDiskLocation', function () { + const dataStores = { + '/vmfs/volumes/uuid-1': { name: 'ds main' }, + 'ds:///vmfs/volumes/uuid-2/': { name: 'ds vsan' }, + } + const relative = { dataStores, currentDataStore: 'ds main', currentPath: 'vm' } + + it('resolves a reference relative to the directory of the vmx', function () { + assert.deepEqual(resolveDiskLocation({ ...relative, filePath: 'vm-000001.vmdk' }), { + dataStore: 'ds main', + path: 'vm/vm-000001.vmdk', + }) + }) + + it('finds the datastore of an absolute reference', function () { + assert.deepEqual(resolveDiskLocation({ ...relative, filePath: '/vmfs/volumes/uuid-1/other/disk.vmdk' }), { + dataStore: 'ds main', + path: 'other/disk.vmdk', + }) + }) + + it('compares normalized paths on both sides', function () { + // a vCenter url against a host path, and the other way around + assert.deepEqual(resolveDiskLocation({ ...relative, filePath: '/vmfs/volumes/uuid-2/vm/disk.vmdk' }), { + dataStore: 'ds vsan', + path: 'vm/disk.vmdk', + }) + assert.deepEqual(resolveDiskLocation({ ...relative, filePath: 'ds:///vmfs/volumes/uuid-1/vm/disk.vmdk' }), { + dataStore: 'ds main', + path: 'vm/disk.vmdk', + }) + }) + + it('does not match a datastore whose url is a prefix of the directory name', function () { + // `/vmfs/volumes/uuid-10` must not be resolved to the datastore `/vmfs/volumes/uuid-1` + assert.throws(() => resolveDiskLocation({ ...relative, filePath: '/vmfs/volumes/uuid-10/vm/disk.vmdk' }), { + code: 'DATASTORE_NOT_FOUND', + }) + }) + + it('skips a datastore whose url is empty', function () { + // an empty prefix matches every absolute path: listed first, it used to shadow every real + // datastore + const withEmpty = { + dataStores: { '': { name: 'ds unmounted' }, 'ds:///': { name: 'ds unmounted too' }, ...dataStores }, + currentDataStore: 'ds main', + currentPath: 'vm', + } + assert.deepEqual(resolveDiskLocation({ ...withEmpty, filePath: '/vmfs/volumes/uuid-1/other/disk.vmdk' }), { + dataStore: 'ds main', + path: 'other/disk.vmdk', + }) + }) + + it('reports an absolute reference on no known datastore', function () { + assert.throws( + () => resolveDiskLocation({ ...relative, filePath: '/vmfs/volumes/uuid-3/vm/disk.vmdk' }), + error => { + assert.equal(error.code, 'DATASTORE_NOT_FOUND') + assert.equal(error.filePath, '/vmfs/volumes/uuid-3/vm/disk.vmdk') + assert.deepEqual(error.dataStoreUrls, ['/vmfs/volumes/uuid-1', 'ds:///vmfs/volumes/uuid-2/']) + return true + } + ) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/_thumbprint.integ.mjs b/@xen-orchestra/vmware-explorer/_thumbprint.integ.mjs new file mode 100644 index 0000000000..f5d863bbd0 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_thumbprint.integ.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict' +import { createServer } from 'node:net' +import { describe, it } from 'node:test' + +import { findFreePort } from './_nbdkit.mjs' +import { getCertificateThumbprint } from './_thumbprint.mjs' + +const noop = () => {} + +// the successful path needs a certificate, so it is only exercised against a real host: what is +// covered here are the two failures which used to freeze or terminate the process +describe('getCertificateThumbprint', function () { + it('does not set the TLS server name when the host is an IP address', async function () { + // RFC 6066 forbids it, and node warns `DeprecationWarning DEP0123` before ignoring it + const warnings = [] + const onWarning = warning => warnings.push(warning) + process.on('warning', onWarning) + + const port = await findFreePort() + try { + await getCertificateThumbprint('127.0.0.1', { port, timeout: 500 }).catch(noop) + // the warning is emitted asynchronously + await new Promise(resolve => setImmediate(resolve)) + assert.deepEqual( + warnings.filter(({ code }) => code === 'DEP0123'), + [] + ) + } finally { + process.off('warning', onWarning) + } + }) + + it('rejects when nothing listens', async function () { + const port = await findFreePort() + + await assert.rejects(getCertificateThumbprint('127.0.0.1', { port, timeout: 2e3 }), error => { + assert.match(error.message, /^can't read the certificate of 127\.0\.0\.1:/) + assert.equal(error.code, 'ECONNREFUSED') + return true + }) + }) + + it('gives up when the handshake never completes', async function () { + // a port accepting the connection then staying silent, as a black holed host does + const server = createServer() + const port = await findFreePort() + await new Promise(resolve => server.listen(port, '127.0.0.1', resolve)) + + try { + await assert.rejects(getCertificateThumbprint('127.0.0.1', { port, timeout: 50 }), { + message: `no TLS handshake with 127.0.0.1:${port} after 50ms`, + }) + } finally { + server.close() + } + }) +}) diff --git a/@xen-orchestra/vmware-explorer/_thumbprint.mjs b/@xen-orchestra/vmware-explorer/_thumbprint.mjs new file mode 100644 index 0000000000..3e2bd25020 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_thumbprint.mjs @@ -0,0 +1,57 @@ +import { connect } from 'node:tls' +import { isIP } from 'node:net' + +/** + * SHA-1 fingerprint of the certificate presented by a TLS server, in the `AA:BB:...` form the + * vddk library expects. + * + * This used to shell out to `openssl s_client` and `openssl x509`, which left a temporary + * directory and a file descriptor behind on every call, had no timeout at all — a black holed port + * froze the whole spawn of the nbdkit server — and threw from inside a callback instead of + * rejecting, terminating the process. + * + * @param {string} host + * @param {object} [options] + * @param {number} [options.port] + * @param {number} [options.timeout] - in ms, for the whole handshake + * @returns {Promise} + */ +export function getCertificateThumbprint(host, { port = 443, timeout = 10e3 } = {}) { + return new Promise((resolve, reject) => { + const socket = connect({ + host, + port, + // the fingerprint is precisely what is being read here, and a self signed certificate is the + // norm on a host + rejectUnauthorized: false, + // needed by a host serving several names, and absent from the openssl call it replaces. + // RFC 6066 forbids an IP address there, and Node warns about it + servername: isIP(host) === 0 ? host : undefined, + timeout, + }) + + const fail = error => { + socket.destroy() + reject(error) + } + + socket.once('secureConnect', () => { + const { fingerprint } = socket.getPeerCertificate() + if (typeof fingerprint !== 'string' || fingerprint.length === 0) { + // nothing to wait for on this path, drop the socket instead of half closing it + return fail(new Error(`the host ${host}:${port} did not present any certificate`)) + } + socket.end() + resolve(fingerprint) + }) + socket.once('timeout', () => { + fail(new Error(`no TLS handshake with ${host}:${port} after ${timeout}ms`)) + }) + socket.once('error', error => { + const wrapped = new Error(`can't read the certificate of ${host}:${port}`) + wrapped.cause = error + wrapped.code = error.code + fail(wrapped) + }) + }) +} diff --git a/@xen-orchestra/vmware-explorer/_vddk.mjs b/@xen-orchestra/vmware-explorer/_vddk.mjs new file mode 100644 index 0000000000..c30787d357 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/_vddk.mjs @@ -0,0 +1,5 @@ +// the vddk library is not redistributable: it has to be downloaded from Broadcom and extracted +// there by `esxi.installVddkLib` +export const VDDK_LIB_DIR = '/usr/local/lib/vddk' +export const VDDK_LIB_PATH = `${VDDK_LIB_DIR}/vmware-vix-disklib-distrib` +export const VDDK_LIB_FILE = `${VDDK_LIB_PATH}/lib64/libvixDiskLib.so` diff --git a/@xen-orchestra/vmware-explorer/checks.mjs b/@xen-orchestra/vmware-explorer/checks.mjs index 745728aec7..1b568f03bb 100644 --- a/@xen-orchestra/vmware-explorer/checks.mjs +++ b/@xen-orchestra/vmware-explorer/checks.mjs @@ -2,6 +2,8 @@ import { exec } from 'node:child_process' import semver from 'semver' import fs from 'node:fs/promises' +import { VDDK_LIB_DIR, VDDK_LIB_FILE } from './_vddk.mjs' + /** * * @returns {Promise} @@ -91,12 +93,11 @@ async function nbdKit() { */ async function vddk() { try { - await fs.stat('/usr/local/lib/vddk/vmware-vix-disklib-distrib/lib64/libvixDiskLib.so') + await fs.stat(VDDK_LIB_FILE) } catch (error) { return { status: 'error', - error: - 'Vddk library is not present or accessible in /usr/local/lib/vddk/ it can be downloaded from https://developer.broadcom.com/sdks/vmware-virtual-disk-development-kit-vddk/latest', + error: `Vddk library is not present or accessible in ${VDDK_LIB_DIR}/ it can be downloaded from https://developer.broadcom.com/sdks/vmware-virtual-disk-development-kit-vddk/latest`, } } return { status: 'success' } diff --git a/@xen-orchestra/vmware-explorer/esxi.fixtures.mjs b/@xen-orchestra/vmware-explorer/esxi.fixtures.mjs new file mode 100644 index 0000000000..f48b69f938 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/esxi.fixtures.mjs @@ -0,0 +1,115 @@ +// helpers shared by the unit tests and the integration tests of the client +import { once } from 'node:events' + +import Esxi from './esxi.mjs' +import { moRef } from './soap/specs.mjs' + +// re-exported: building the fixtures with the same helper as the production code means they cannot +// drift from the shape the tests are supposed to assert on +export { moRef } + +// a page of a RetrievePropertiesEx response +export const page = (objects, token) => ({ returnval: { objects, token } }) + +// two datacenters and three datastores, shaped as the SOAP library returns them +export const DATACENTERS = [ + { + obj: moRef('Datacenter', 'datacenter-1'), + propSet: [ + { name: 'name', val: { $value: 'dc-main', attributes: { 'xsi:type': 'string' } } }, + { + name: 'datastore', + val: { + attributes: { 'xsi:type': 'ArrayOfManagedObjectReference' }, + ManagedObjectReference: [moRef('Datastore', 'datastore-11'), moRef('Datastore', 'datastore-12')], + }, + }, + ], + }, + { + obj: moRef('Datacenter', 'datacenter-2'), + propSet: [ + { name: 'name', val: { $value: 'dc-other' } }, + // a single reference is not wrapped in an array + { name: 'datastore', val: { attributes: {}, ManagedObjectReference: moRef('Datastore', 'datastore-21') } }, + ], + }, +] + +export const DATASTORES = [ + // a single property is not wrapped in an array either + { obj: moRef('Datastore', 'datastore-11'), propSet: { name: 'name', val: { $value: 'ds main' } } }, + { obj: moRef('Datastore', 'datastore-12'), propSet: [{ name: 'name', val: { $value: 'ds2' } }] }, + { obj: moRef('Datastore', 'datastore-21'), propSet: [{ name: 'name', val: { $value: 'ds3' } }] }, +] + +export const INVENTORY_RESPONSES = { + CreateContainerView: () => ({ returnval: moRef('ContainerView', 'session[42]view-1') }), + DestroyView: () => ({}), + RetrievePropertiesEx: ({ specSet }) => page(specSet[0].propSet[0].type === 'Datacenter' ? DATACENTERS : DATASTORES), +} + +/** + * Records the calls made to the host and answers them with canned responses. + * + * A response is a function of the arguments of the call; an absent one answers `undefined`, as the + * host does when nothing matches. + */ +export class FakeVimClient { + constructor(responses = INVENTORY_RESPONSES) { + this.calls = [] + this.responses = responses + this.serviceContent = { + propertyCollector: moRef('PropertyCollector', 'propertyCollector'), + rootFolder: moRef('Folder', 'group-d1'), + viewManager: moRef('ViewManager', 'ViewManager'), + } + this.closed = false + } + + async connect() {} + + async call(method, args) { + this.calls.push({ method, args }) + return this.responses[method]?.(args) + } + + async close() { + this.closed = true + } + + get methods() { + return this.calls.map(({ method }) => method) + } + + callsTo(method) { + return this.calls.filter(call => call.method === method) + } +} + +/** a minimal `fetch` response */ +export const response = ({ status = 200, statusText = 'OK', headers = {}, body = '' } = {}) => { + const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body) + return { + status, + statusText, + headers: new Headers({ 'content-length': String(buffer.length), ...headers }), + body: { cancel: async () => {} }, + arrayBuffer: async () => buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), + text: async () => buffer.toString('utf8'), + } +} + +/** connects with the inventory above, then hands the client over to the test */ +export const connectedEsxi = async ({ responses, ...options } = {}) => { + const vimClient = new FakeVimClient() + const esxi = new Esxi('esxi.test', 'user', 'password', true, { ...options, vimClient }) + await once(esxi, 'ready') + + if (responses !== undefined) { + vimClient.responses = { ...INVENTORY_RESPONSES, ...responses } + } + vimClient.calls.length = 0 + + return { esxi, vimClient } +} diff --git a/@xen-orchestra/vmware-explorer/esxi.integ.mjs b/@xen-orchestra/vmware-explorer/esxi.integ.mjs new file mode 100644 index 0000000000..eb36039eb8 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/esxi.integ.mjs @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict' +import { createServer } from 'node:net' +import { describe, it } from 'node:test' +import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' + +import { connectedEsxi } from './esxi.fixtures.mjs' + +// these bind a real port and write real temporary files, unlike the rest of the suite +describe('nbdkit servers', function () { + /** + * Stands in for nbdkit: it listens on the port it is given, so that the readiness probe of the + * server under test is exercised for real. + */ + const fakeNbdkit = ({ exitWith, failWith } = {}) => { + const spawned = [] + const spawn = (command, args) => { + const child = new EventEmitter() + child.exitCode = null + child.signalCode = null + child.stdout = new PassThrough() + child.stderr = new PassThrough() + child.kill = signal => { + child.signalCode = signal ?? 'SIGTERM' + child.exitCode = 0 + entry.server?.close() + setImmediate(() => child.emit('exit', 0, child.signalCode)) + return true + } + + const port = Number(args.find(argument => argument.startsWith('--port=')).slice('--port='.length)) + const entry = { args, child, command, port } + spawned.push(entry) + + if (failWith !== undefined) { + setImmediate(() => child.emit('error', failWith)) + } else if (exitWith !== undefined) { + // nbdkit exits on a bad thumbprint or a missing vddk library, without ever listening + setImmediate(() => { + child.exitCode = exitWith + child.emit('exit', exitWith, null) + }) + } else { + entry.server = createServer() + entry.server.listen(port, '127.0.0.1') + } + return child + } + return { spawn, spawned } + } + + const nbdkitEsxi = async options => { + const { spawn, spawned } = fakeNbdkit(options) + const { esxi } = await connectedEsxi({ spawn }) + // the thumbprint is the only step of a spawn which needs the real host + esxi.getServerThumbprint = async () => 'AA:BB:CC' + return { esxi, spawned } + } + + it('spawns one server per disk, and shares it', async function () { + const { esxi, spawned } = await nbdkitEsxi() + + const [first, second] = await Promise.all([ + esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk'), + esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk'), + ]) + + // the promise is memoized: two concurrent calls used to spawn two servers, orphaning one + assert.equal(spawned.length, 1) + assert.equal(first.value, second.value) + const server = first.value + assert.equal(server.nbdInfos.exportname, '[ds] vm/vm.vmdk') + assert.equal(server.nbdInfos.port, spawned[0].port) + + // the server belongs to both of them: the first to let go must not stop it + await first.dispose() + assert.equal(server.process.exitCode, null) + + await second.dispose() + assert.equal(server.process.exitCode, 0) + + await esxi.close() + }) + + it('spawns a new server after the previous one was killed', async function () { + const { esxi, spawned } = await nbdkitEsxi() + + const first = await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + const firstServer = first.value + await first.dispose() + const second = await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + + // the entry used to be left in the map, so this handed out the dead process of a closed port + assert.equal(spawned.length, 2) + assert.notEqual(firstServer.nbdInfos.port, second.value.nbdInfos.port) + assert.equal(firstServer.process.exitCode, 0) + + await esxi.close() + }) + + it('forgets a server which died on its own', async function () { + const { esxi, spawned } = await nbdkitEsxi() + + const first = await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + spawned[0].server.close() + first.value.process.exitCode = 1 + first.value.process.emit('exit', 1, null) + await first.value.died + + await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + + assert.equal(spawned.length, 2) + + await esxi.close() + }) + + it('reports a missing nbdkit instead of terminating the process', async function () { + const error = new Error('spawn nbdkit ENOENT') + error.code = 'ENOENT' + const { esxi, spawned } = await nbdkitEsxi({ failWith: error }) + + // without an 'error' listener on the child process, this used to be an uncaught event + await assert.rejects(esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk'), { code: 'ENOENT' }) + + // the failure is not memoized either + await assert.rejects(esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk'), { code: 'ENOENT' }) + assert.equal(spawned.length, 2) + }) + + it('reports an nbdkit which exited before being ready', async function () { + const { esxi, spawned } = await nbdkitEsxi({ exitWith: 1 }) + + // waiting for the port would otherwise burn the whole readiness timeout + await assert.rejects(esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk'), { + code: 'NBDKIT_EXITED', + message: /^nbdkit exited with code 1 before being ready/, + }) + assert.equal(spawned.length, 1) + + await esxi.close() + }) + + it('never passes the password on the command line', async function () { + const { esxi, spawned } = await nbdkitEsxi() + + await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + + assert.equal( + spawned[0].args.some(argument => argument.includes('password')), + true + ) + assert.equal( + spawned[0].args.some(argument => argument.includes('password=password')), + false + ) + + await esxi.close() + }) + + it('kills the remaining servers when closing', async function () { + const { esxi, spawned } = await nbdkitEsxi() + + // never disposed: closing is the safety net for a caller which leaked its disposable + const disposable = await esxi.getNbdServer('vm-1', '[ds] vm/vm.vmdk') + const server = disposable.value + await esxi.close() + + assert.equal(server.process.exitCode, 0) + assert.equal(spawned.length, 1) + + // disposing after the close has nothing left to do, and must not throw + await disposable.dispose() + }) +}) diff --git a/@xen-orchestra/vmware-explorer/esxi.mjs b/@xen-orchestra/vmware-explorer/esxi.mjs index 498eb65199..386d59ba2b 100644 --- a/@xen-orchestra/vmware-explorer/esxi.mjs +++ b/@xen-orchestra/vmware-explorer/esxi.mjs @@ -1,15 +1,23 @@ -import { Client } from '@vates/node-vsphere-soap' +import { asyncEach } from '@vates/async-each' import { createLogger } from '@xen-orchestra/log' import { dirname, join } from 'node:path' import { EventEmitter } from 'node:events' -import { strictEqual, notStrictEqual } from 'node:assert' +import { Disposable, pTimeout } from 'promise-toolbox' +import { setTimeout as delay } from 'node:timers/promises' import { Agent } from 'undici' +import { findFreePort, formatNbdkitArgs, waitForPort } from './_nbdkit.mjs' +import { resolveDiskLocation } from './_paths.mjs' +import { getCertificateThumbprint } from './_thumbprint.mjs' +import { VDDK_LIB_PATH } from './_vddk.mjs' +import { COWD_HEADER_LENGTH, grainDirectoryToDataMap, parseCowdHeader } from './parsers/cowd.mjs' import parseVmdk from './parsers/vmdk.mjs' import parseVmsd from './parsers/vmsd.mjs' import parseVmx from './parsers/vmx.mjs' -import xml2js from 'xml2js' -import { exec, spawn } from 'node:child_process' +import { asArray, normalizeSoapValue } from './soap/normalize.mjs' +import { moRef, objectSpec, propertyFilterSpec, propertySpec, retrieveOptions, traversalSpec } from './soap/specs.mjs' +import { VimClient } from './soap/VimClient.mjs' +import { spawn } from 'node:child_process' import NbdClient from '@vates/nbd-client' import { tmpdir } from 'node:os' @@ -17,22 +25,103 @@ import fs from 'node:fs/promises' const { info, warn } = createLogger('xo:vmware-explorer:esxi') -export const VDDK_LIB_DIR = '/usr/local/lib/vddk' -export const VDDK_LIB_PATH = `${VDDK_LIB_DIR}/vmware-vix-disklib-distrib` -let nbdPort = 11000 +export { VDDK_LIB_DIR, VDDK_LIB_PATH } from './_vddk.mjs' + +const DEFAULT_DOWNLOAD_RETRIES = 4 +// a vmdk descriptor is a text file of a few hundred bytes, but a long parentFileNameHint pushes +// its extent line further +const DESCRIPTOR_READ_LENGTH = 4096 +// a vmdk descriptor is a small file, but the /folder endpoint of a host is not fast +const DESCRIPTOR_CONCURRENCY = 4 +const DEFAULT_HEADERS_TIMEOUT = 60e3 +const DEFAULT_RETRY_DELAY = 2e3 +// every caller of `#waitForTaskEnd` passes its own deadline, this is only the guardrail for the +// next one: the timeout is compared against, so an undefined one would poll for ever +const DEFAULT_TASK_TIMEOUT = 60e3 +const MAX_RETRY_DELAY = 30e3 +const MAX_TASK_POLL_DELAY = 5e3 +const NBDKIT_KILL_TIMEOUT = 10e3 +// connecting the vddk library to the host can be slow +const NBDKIT_READY_TIMEOUT = 60e3 + +// a failure which will not fix itself must not be retried: a missing file, a rejected +// authentication or a programming error only delay the report of the real problem +const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]) +const SESSION_EXPIRED = 'ESXI_SESSION_EXPIRED' +const RETRYABLE_CODES = new Set([ + SESSION_EXPIRED, + 'ECONNABORTED', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ENOTFOUND', + 'EPIPE', + 'ETIMEDOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_SOCKET', +]) + +function isRetryableError(error) { + // `fetch` reports a network failure as `TypeError: fetch failed`, with the real error as cause + if (RETRYABLE_CODES.has(error?.code) || RETRYABLE_CODES.has(error?.cause?.code)) { + return true + } + const status = error?.cause?.status + if (status !== undefined) { + return RETRYABLE_STATUS.has(status) + } + // the host did not send its response headers in time + return error?.name === 'TimeoutError' +} + +const noop = () => {} + +// the options change what the server exports, so they are part of its identity +const nbdServerKey = (vmId, diskPath, { compression, singleLink, threads }) => + JSON.stringify([vmId, diskPath, singleLink, threads, compression]) + +// `TaskInfo.error` is a LocalizedMethodFault: the concrete fault type is the `xsi:type` of its +// `fault` element, and the parser may expose it at either level depending on the response +const taskFaultType = error => error?.fault?.attributes?.['xsi:type'] ?? error?.attributes?.['xsi:type'] +const taskFaultMessage = error => error?.localizedMessage ?? error?.fault?.localizedMessage + export default class Esxi extends EventEmitter { - #client + #connected #cookies #dcPaths // map datastore name => datacenter name + #fetchImpl #host #httpsAgent #user #password - #ready = false + #spawn + #thumbprint + #vimClient #nbdServers = new Map() - constructor(host, user, password, sslVerify) { + /** + * @param {string} host + * @param {string} user + * @param {string} password + * @param {boolean} sslVerify + * @param {object} [options] + * @param {typeof globalThis.fetch} [options.fetch] - injectable fetch implementation, for tests + * @param {typeof spawn} [options.spawn] - injectable process spawner, for tests + * @param {object} [options.vimClient] - injectable SOAP client, for tests + */ + constructor( + host, + user, + password, + sslVerify, + { fetch: fetchImplementation, spawn: spawnImplementation, vimClient } = {} + ) { super() + this.#fetchImpl = fetchImplementation ?? globalThis.fetch + this.#spawn = spawnImplementation ?? spawn this.#host = host.trim() this.#user = user this.#password = password @@ -44,139 +133,237 @@ export default class Esxi extends EventEmitter { }) } - this.#client = new Client(host, user, password, sslVerify) - this.#client.once('ready', async () => { - try { - // this.#ready is set to true to allow the this.search query to go through - // this means that the server is connected and can answer API queries - // you won't be able to download a file as long a the 'ready' event is not emitted - this.#ready = true - await this.#computeDatacenters() - this.emit('ready') - } catch (error) { - this.emit('error', error) - } - }) - this.#client.on('error', err => { - this.emit('error', err) - }) - } + this.#vimClient = + vimClient ?? new VimClient(host, user, password, sslVerify, { onError: error => this.#onError(error) }) - async #computeDatacenters() { - this.#dcPaths = {} - // the datastore property is a collection of datastore id - const res = await this.search('Datacenter', ['name', 'datastore']) - await Promise.all( - Object.values(res).map(async ({ datastore, name }) => { - if (datastore.ManagedObjectReference === undefined) { - return - } - await Promise.all( - datastore.ManagedObjectReference.map(async ({ $value }) => { - // get the datastore name - const res = await this.fetchProperty('Datastore', $value, 'name') - this.#dcPaths[res._] = name - }) - ) - }) + // every method awaits this promise, the event is kept for the callers relying on it + this.#connected = this.#connect() + this.#connected.then( + () => this.emit('ready'), + error => this.#onError(error) ) } - #findDatacenter(dataStore) { + async #connect() { + await this.#vimClient.connect() + // the datacenter of a datastore is needed to download a file from it + await this.#computeDatacenters() + } + + #onError(error) { + // an 'error' event without any listener terminates the process, and a caller waiting for + // 'ready' has no reason to still be listening once connected + if (this.listenerCount('error') > 0) { + this.emit('error', error) + } else { + warn('esxi client error', { error, host: this.#host }) + } + } + + /** + * Closes the session on the host. + * + * @returns {Promise} + */ + async close() { try { - notStrictEqual(this.#dcPaths, undefined) - notStrictEqual(this.#dcPaths[dataStore], undefined) - } catch (error) { + // the servers would only be reaped by `--exit-with-parent`, i.e. when this process ends + await asyncEach([...this.#nbdServers.keys()], key => this.#killNbdServerByKey(key), { + concurrency: 4, + // every server must be tried: `asyncEach` stops on the first error by default, which would + // leave the other ones running + stopOnError: false, + }) + } finally { + // a server refusing to die must not leave the session open on the host, nor the sockets of + // the agent + await this.#vimClient.close() + await this.#httpsAgent?.close() + } + } + + async #computeDatacenters() { + // the names of the datastores are read in one call instead of one per datastore: a host with + // many datastores used to issue as many requests, all in flight at the same time + const [datacenters, datastores] = await Promise.all([ + // the datastore property is a collection of datastore ids + this.search('Datacenter', ['name', 'datastore']), + this.search('Datastore', ['name']), + ]) + + const dcPaths = {} + for (const { datastore, name: datacenterName } of Object.values(datacenters)) { + for (const reference of asArray(datastore?.ManagedObjectReference)) { + const datastoreName = datastores[reference.$value]?.name + if (datastoreName === undefined) { + warn('a datastore of a datacenter is not listed', { datacenterName, datastore: reference.$value }) + continue + } + dcPaths[datastoreName] = datacenterName + } + } + this.#dcPaths = dcPaths + } + + #findDatacenter(dataStore) { + const dcPath = this.#dcPaths?.[dataStore] + if (dcPath === undefined) { warn("can't find datacenter for datastore", { datacenters: this.#dcPaths, dataStore }) + // an assertion error used to be thrown, naming neither the datastore nor the known ones + const error = new Error(`can't find the datacenter of the datastore ${dataStore}`) + error.code = 'DATACENTER_NOT_FOUND' + error.dataStore = dataStore + error.dataStores = Object.keys(this.#dcPaths ?? {}) throw error } - return this.#dcPaths[dataStore] + return dcPath } - #exec(cmd, args) { - strictEqual(this.#ready, true) - const client = this.#client - return new Promise(function (resolve, reject) { - client.once('error', function (error) { - client.off('result', resolve) - reject(error) - }) - client.runCommand(cmd, args).once('result', function () { - client.off('error', reject) - resolve(...arguments) - }) - }) - } - - async #fetch(url, headers = {}, signal) { - if (this.#cookies) { + async #fetch(url, { range, signal, headersTimeout = DEFAULT_HEADERS_TIMEOUT } = {}) { + const headers = {} + if (this.#cookies !== undefined) { headers.cookie = this.#cookies } else { headers.Authorization = 'Basic ' + Buffer.from(this.#user + ':' + this.#password).toString('base64') } - const res = await fetch(url, { - dispatcher: this.#httpsAgent, - method: 'GET', - headers, - highWaterMark: 10 * 1024 * 1024, - signal, - }) + if (range !== undefined) { + headers.Range = 'bytes=' + range + } + + // the timeout covers the response headers only: the body of a disk legitimately takes hours, + // an abort signal outliving the headers would cut the stream in the middle + const controller = new AbortController() + const timer = setTimeout(() => { + controller.abort(new DOMException(`no response header after ${headersTimeout}ms`, 'TimeoutError')) + }, headersTimeout) + + let res + try { + res = await this.#fetchImpl(url, { + dispatcher: this.#httpsAgent, + method: 'GET', + headers, + // the caller keeps the ability to abort the body + signal: signal === undefined ? controller.signal : AbortSignal.any([controller.signal, signal]), + }) + } finally { + clearTimeout(timer) + } + if (res.status < 200 || res.status >= 300) { + // the body must be consumed or the connection is never released, and it usually explains the + // failure better than the status alone + const body = await res.text().catch(() => undefined) const error = new Error(res.status + ' ' + res.statusText + ' ' + url) - error.cause = res + error.cause = { status: res.status, statusText: res.statusText, url: String(url), body: body?.slice(0, 2048) } + if ((res.status === 401 || res.status === 403) && headers.cookie !== undefined) { + // the session expired, which happens on an import lasting hours: forget it so that the + // next attempt authenticates again + this.#cookies = undefined + error.code = SESSION_EXPIRED + } throw error } - if (res.headers['set-cookie']) { - this.#cookies = res.headers['set-cookie'].map(cookie => cookie.split(';')[0]).join('; ') + + if (range !== undefined && res.status !== 206) { + // A host answers the whole file when the range covers it entirely: `/folder` does it for a + // vmdk descriptor of a few hundred bytes read as `0-511`. That is harmless, since such an + // answer starts at the first byte and holds no more than what was asked. + // + // Anything else is not the requested range, and reading it as if it were would silently use + // the wrong offset, or as much memory as the file is big + const [start, end] = range.split('-').map(Number) + const rawLength = res.headers.get('content-length') + const length = Number(rawLength) + // an answer whose size is unknown ( chunked ) cannot be checked, and `Number(null)` is 0 + if (start !== 0 || rawLength === null || !Number.isInteger(length) || length > end - start + 1) { + await res.body?.cancel() + const error = new Error( + `the range ${range} was ignored by the host (status ${res.status}, ${res.headers.get('content-length')} bytes) ${url}` + ) + error.code = 'RANGE_IGNORED' + error.cause = { status: res.status, url: String(url), range, length } + throw error + } } + + const cookies = res.headers.getSetCookie?.() ?? [] + if (cookies.length > 0) { + // reusing the session saves a full authentication on every range request + this.#cookies = cookies.map(cookie => cookie.split(';')[0]).join('; ') + } + return res } - async #download(dataStore, path, range, signal) { - strictEqual(this.#ready, true) + async #download(dataStore, path, { range, signal } = {}) { + // the datacenter of the datastore is only known once connected + await this.#connected const url = new URL('https://localhost') url.host = this.#host url.pathname = '/folder/' + path url.searchParams.set('dcPath', this.#findDatacenter(dataStore)) url.searchParams.set('dsName', dataStore) - const headers = {} - if (range) { - headers['content-type'] = 'multipart/byteranges' - headers.Range = 'bytes=' + range - } - return this.#fetch(url, headers, signal) + + return this.#fetch(url, { range, signal }) } - async download(dataStore, path, range, signal) { - let tries = 5 - let lastError - while (tries > 0) { + /** + * Downloads a file of a datastore, or a range of it. + * + * @param {string} dataStore - name of the datastore + * @param {string} path - path of the file in the datastore + * @param {object} [options] + * @param {string} [options.range] - inclusive byte range, e.g. `0-511` + * @param {number} [options.retries] - how many times a retryable failure is retried + * @param {number} [options.retryDelay] - in ms, doubled at every attempt + * @param {AbortSignal} [options.signal] + * @returns {Promise} + */ + async download( + dataStore, + path, + { range, retries = DEFAULT_DOWNLOAD_RETRIES, retryDelay = DEFAULT_RETRY_DELAY, signal } = {} + ) { + for (let attempt = 0; ; attempt++) { try { - const res = await this.#download(dataStore, path, range, signal) - return res + return await this.#download(dataStore, path, { range, signal }) } catch (error) { signal?.throwIfAborted() - warn('got error , will retry in 2 seconds', { error }) - lastError = error + // retrying a missing file or a rejected authentication only delays the failure + if (attempt >= retries || !isRetryableError(error)) { + throw error + } + const wait = Math.round( + Math.min(retryDelay * 2 ** attempt, MAX_RETRY_DELAY) * (0.5 + Math.random() / 2) // jitter + ) + warn('download failed, will retry', { attempt: attempt + 1, dataStore, delay: wait, error, path, range }) + await delay(wait, undefined, { signal }) } - await new Promise(resolve => setTimeout(() => resolve(), 2000)) - tries-- } - - throw lastError } // inspired from https://github.com/reedog117/node-vsphere-soap/blob/master/test/vsphere-soap.test.js#L95 - async search(type, properties) { - // search types are limited to "ComputeResource", "Datacenter", "Datastore", "DistributedVirtualSwitch", "Folder", "HostSystem", "Network", "ResourcePool", "VirtualMachine"} - // from https://github.com/vmware/govmomi/issues/2595#issuecomment-966604502 + /** + * Lists the objects of a type with the given properties. + * + * @param {string} type - "ComputeResource", "Datacenter", "Datastore", "DistributedVirtualSwitch", + * "Folder", "HostSystem", "Network", "ResourcePool" or "VirtualMachine" + * ( from https://github.com/vmware/govmomi/issues/2595#issuecomment-966604502 ) + * @param {ReadonlyArray} properties + * @param {object} [options] + * @param {number} [options.maxObjects] - objects per page + * @returns {Promise>} properties indexed by managed object reference + */ + async search(type, properties, { maxObjects = 100 } = {}) { + await this.#vimClient.connect() // get property collector - const propertyCollector = this.#client.serviceContent.propertyCollector + const propertyCollector = this.#vimClient.serviceContent.propertyCollector // get view manager - const viewManager = this.#client.serviceContent.viewManager + const viewManager = this.#vimClient.serviceContent.viewManager // get root folder - const rootFolder = this.#client.serviceContent.rootFolder - let result = await this.#exec('CreateContainerView', { + const rootFolder = this.#vimClient.serviceContent.rootFolder + let result = await this.#vimClient.call('CreateContainerView', { _this: viewManager, container: rootFolder, type: [type], @@ -186,84 +373,80 @@ export default class Esxi extends EventEmitter { // build all the data structures needed to query all the vm names const containerView = result.returnval - const objectSpec = { - attributes: { 'xsi:type': 'ObjectSpec' }, // setting attributes xsi:type is important or else the server may mis-recognize types! - obj: containerView, - skip: true, - selectSet: [ - { - attributes: { 'xsi:type': 'TraversalSpec' }, - name: 'traverseEntities', - type: 'ContainerView', - path: 'view', - skip: false, - }, + // the specs are built in schema order, and the `xsi:type` are mandatory or else the server may + // mis-recognize the types + const filterSpec = propertyFilterSpec({ + // every path has the same type, so one spec carries them all + propSet: [propertySpec({ type, pathSet: properties })], + objectSet: [ + objectSpec({ + obj: containerView, + skip: true, + selectSet: [traversalSpec({ name: 'traverseEntities', type: 'ContainerView', path: 'view', skip: false })], + }), ], - } + }) - const propertyFilterSpec = { - attributes: { 'xsi:type': 'PropertyFilterSpec' }, - propSet: properties.map(p => ({ - attributes: { 'xsi:type': 'PropertySpec' }, - type, - pathSet: [p], - })), - objectSet: [objectSpec], - } - - let token const objects = {} - do { - if (token !== undefined) { - result = await this.#exec('ContinueRetrievePropertiesEx', { - _this: propertyCollector, - token, - }) - } else { - result = await this.#exec('RetrievePropertiesEx', { - _this: propertyCollector, - specSet: [propertyFilterSpec], - options: { attributes: { type: 'RetrieveOptions' } }, - }) - } - - const returnObj = Array.isArray(result.returnval.objects) ? result.returnval.objects : [result.returnval.objects] - returnObj.forEach(({ obj, propSet }) => { - objects[obj.$value] = {} - propSet = Array.isArray(propSet) ? propSet : [propSet] - propSet.forEach(({ name, val }) => { - // don't care about the type for now - delete val.attributes - // a scalar value : simplify it - if (val.$value) { - objects[obj.$value][name] = val.$value - } else { - objects[obj.$value][name] = val - } - }) + // the token of the page being retrieved, as long as it has not been consumed + let pendingToken + try { + result = await this.#vimClient.call('RetrievePropertiesEx', { + _this: propertyCollector, + specSet: [filterSpec], + options: retrieveOptions({ maxObjects }), }) - token = result.returnval.token - } while (token) + for (;;) { + // `returnval` is absent when no object matches + const returnval = result?.returnval + if (returnval === undefined) { + break + } + + for (const { obj, propSet } of asArray(returnval.objects)) { + // an object can be split across pages + const objectProperties = (objects[obj.$value] ??= {}) + // `propSet` is absent for an object with no readable property + for (const { name, val } of asArray(propSet)) { + objectProperties[name] = normalizeSoapValue(val) + } + } + + pendingToken = returnval.token + if (pendingToken === undefined) { + break + } + result = await this.#vimClient.call('ContinueRetrievePropertiesEx', { + _this: propertyCollector, + token: pendingToken, + }) + } + } finally { + if (pendingToken !== undefined) { + // the retrieval was not consumed entirely, the server keeps its results until then + await this.#vimClient + .call('CancelRetrievePropertiesEx', { _this: propertyCollector, token: pendingToken }) + .catch(error => warn('failed to cancel the property retrieval', { error, token: pendingToken })) + } + // a view is a server side resource: not destroying it leaks one per search + await this.#vimClient + .call('DestroyView', { _this: containerView }) + .catch(error => warn('failed to destroy the container view', { error, type })) + } return objects } - async #inspectVmdk(dataStores, currentDataStore, currentPath, filePath) { - let diskDataStore, diskPath - if (filePath.startsWith('/')) { - // disk is on another datastore - Object.keys(dataStores).forEach(dataStoreUrl => { - if (filePath.startsWith(dataStoreUrl)) { - diskDataStore = dataStores[dataStoreUrl].name - diskPath = filePath.substring(dataStoreUrl.length + 1) - } - }) - } else { - diskDataStore = currentDataStore - diskPath = currentPath + '/' + filePath - } - const vmdkRes = await this.download(diskDataStore, diskPath) + async #inspectVmdk(dataStores, currentDataStore, currentPath, filePath, { signal } = {}) { + const { dataStore: diskDataStore, path: diskPath } = resolveDiskLocation({ + dataStores, + currentDataStore, + currentPath, + filePath, + }) + + const vmdkRes = await this.download(diskDataStore, diskPath, { signal }) const text = await vmdkRes.text() const parsed = parseVmdk(text) @@ -279,61 +462,83 @@ export default class Esxi extends EventEmitter { async getAllVmMetadata() { const datas = await this.search('VirtualMachine', ['config', 'storage', 'runtime', 'layoutEx']) - return Object.keys(datas) - .map(id => { - const { config, layoutEx, storage, runtime } = datas[id] - if (storage === undefined || config === undefined) { - return undefined - } - // vsan , maybe raw disk , that forbid access to a direct vmdk - // descriptor may exist though with a .vmdk extension - let hasAllExtentsListed = true - - // structure of layoutEx is described in https://developer.vmware.com/apis/1720/ - layoutEx?.disk?.forEach(disk => { - // we can stop, even if only one disk is missing an extent - hasAllExtentsListed && - disk.chain?.forEach(({ fileKey: fileKeys }) => { - // look for the disk extent data , not the descriptor - const fileExtent = layoutEx.file.find(file => { - return fileKeys.includes(file.key) && file.type === 'diskExtent' - }) - hasAllExtentsListed = hasAllExtentsListed && fileExtent !== undefined - }) - }) - const perDatastoreUsage = Array.isArray(storage.perDatastoreUsage) - ? storage.perDatastoreUsage - : [storage.perDatastoreUsage] - return { + const metadata = [] + for (const [id, { config, layoutEx, runtime, storage }] of Object.entries(datas)) { + // an incomplete VM, e.g. one being created: the fields below used to be dereferenced anyway, + // which failed the whole listing. `config` can be reported without its `hardware` yet + if (config?.hardware === undefined || runtime?.powerState === undefined || storage === undefined) { + warn('ignoring a VM whose properties are incomplete', { id, - hasAllExtentsListed, - nameLabel: config.name, - memory: +config.hardware.memoryMB * 1024 * 1024, - nCpus: +config.hardware.numCPU, - guestToolsInstalled: false, - firmware: config.firmware === 'efi' ? 'uefi' : config.firmware, // bios or uefi - powerState: runtime.powerState, - storage: perDatastoreUsage.reduce( - (prev, curr) => { - return { - used: prev.used + +(curr?.committed ?? 0), - free: prev.free + +(curr?.uncommitted ?? 0), - } - }, - { used: 0, free: 0 } - ), + missing: { + 'config.hardware': config?.hardware === undefined, + 'runtime.powerState': runtime?.powerState === undefined, + storage: storage === undefined, + }, + }) + continue + } + + // vsan , maybe raw disk , that forbid access to a direct vmdk + // descriptor may exist though with a .vmdk extension + // structure of layoutEx is described in https://developer.vmware.com/apis/1720/ + const layoutFiles = asArray(layoutEx?.file) + let hasAllExtentsListed = true + for (const disk of asArray(layoutEx?.disk)) { + for (const link of asArray(disk.chain)) { + const fileKeys = asArray(link.fileKey) + // look for the disk extent data , not the descriptor + if (!layoutFiles.some(file => fileKeys.includes(file.key) && file.type === 'diskExtent')) { + hasAllExtentsListed = false + break + } } + // one disk missing an extent is enough + if (!hasAllExtentsListed) { + break + } + } + + metadata.push({ + id, + hasAllExtentsListed, + nameLabel: config.name, + memory: +config.hardware.memoryMB * 1024 * 1024, + nCpus: +config.hardware.numCPU, + guestToolsInstalled: false, + firmware: config.firmware === 'efi' ? 'uefi' : config.firmware, // bios or uefi + powerState: runtime.powerState, + storage: asArray(storage.perDatastoreUsage).reduce( + (prev, curr) => { + return { + used: prev.used + +(curr?.committed ?? 0), + free: prev.free + +(curr?.uncommitted ?? 0), + } + }, + { used: 0, free: 0 } + ), }) - .filter(_ => _ !== undefined) + } + return metadata } - async getTransferableVmMetadata(vmId) { + async getTransferableVmMetadata(vmId, { signal } = {}) { const [config, runtime] = await Promise.all([ - this.fetchProperty('VirtualMachine', vmId, 'config'), - this.fetchProperty('VirtualMachine', vmId, 'runtime'), + this.#retrieveProperty('VirtualMachine', vmId, 'config', { signal }), + this.#retrieveProperty('VirtualMachine', vmId, 'runtime', { signal }), ]) - const [, dataStore, vmxPath] = config.files[0].vmPathName[0].match(/^\[(.*)\] (.+.vmx)$/) - const res = await this.download(dataStore, vmxPath) + + const vmPathName = config.files.vmPathName + const matches = vmPathName.match(/^\[(.*)\] (.+\.vmx)$/) + if (matches === null) { + // destructuring the null used to throw a TypeError naming nothing + const error = new Error(`can't parse the path of the vmx of the VM ${vmId}: ${vmPathName}`) + error.code = 'BAD_VMX_PATH' + error.vmId = vmId + throw error + } + const [, dataStore, vmxPath] = matches + + const res = await this.download(dataStore, vmxPath, { signal }) const vmx = parseVmx(await res.text()) // list datastores const dataStores = {} @@ -341,21 +546,23 @@ export default class Esxi extends EventEmitter { dataStores[summary.url] = summary }) - const disks = [] + const diskReferences = [] const networks = [] let cdrom = false for (const key of Object.keys(vmx)) { - const matches = key.match(/^(scsi|ide|ethernet|sata)[0-9]+$/) - if (matches === null) { + const channelMatches = key.match(/^(scsi|ide|ethernet|sata)[0-9]+$/) + if (channelMatches === null) { continue } - const channelType = matches[1] + const channelType = channelMatches[1] if (channelType === 'ide' || channelType === 'scsi' || channelType === 'sata' /* cdrom */) { const diskChannel = vmx[key] - for (const diskIndex in Object.values(diskChannel)) { - const disk = diskChannel[diskIndex] - if (typeof disk !== 'object') { + // the indexes of the channel are its own: iterating the compacted values used to label a + // channel holding only `scsi0:1` as `scsi0:0`, and `node` is the disk identity used to + // build the chains and to order the VBDs + for (const [diskIndex, disk] of Object.entries(diskChannel)) { + if (typeof disk !== 'object' || disk === null) { continue } if (disk.deviceType?.match(/cdrom/i)) { @@ -367,10 +574,7 @@ export default class Esxi extends EventEmitter { continue } - disks.push({ - ...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), disk.fileName)), - node: `${key}:${diskIndex}`, - }) + diskReferences.push({ fileName: disk.fileName, node: `${key}:${diskIndex}` }) } } else if (channelType === 'ethernet') { const ethernet = vmx[key] @@ -382,33 +586,61 @@ export default class Esxi extends EventEmitter { }) } } + const inspect = fileName => this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), fileName, { signal }) + + // one descriptor to download per disk: read serially, a long chain added a full round trip per + // snapshot and per disk + const disks = new Array(diskReferences.length) + await asyncEach( + diskReferences, + async ({ fileName, node }, index) => { + disks[index] = { ...(await inspect(fileName)), node } + }, + { concurrency: DESCRIPTOR_CONCURRENCY, signal } + ) + let snapshots try { - const vmsd = await (await this.download(dataStore, vmxPath.replace(/\.vmx$/, '.vmsd'))).text() - snapshots = parseVmsd(vmsd) + const vmsd = await this.download(dataStore, vmxPath.replace(/\.vmx$/, '.vmsd'), { signal }) + snapshots = parseVmsd(await vmsd.text()) + } catch (error) { + if (error.cause?.status !== 404) { + // an unreadable vmsd used to be silently reported as "no snapshot", which turns a delta + // transfer into a full one without telling anybody + throw error + } + info('no vmsd file, the VM has no snapshot', { vmId }) + } - for (const snapshotIndex in snapshots?.snapshots) { - const snapshot = snapshots.snapshots[snapshotIndex] - for (const diskIndex in snapshot.disks) { - const fileName = snapshot.disks[diskIndex].fileName - snapshot.disks[diskIndex] = { - node: snapshot.disks[diskIndex]?.node, // 'scsi0:0' , 'ide0:0', ..., - ...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), fileName)), - } + if (snapshots !== undefined) { + const snapshotDisks = [] + for (const snapshot of snapshots.snapshots ?? []) { + for (const diskIndex of Object.keys(snapshot.disks ?? {})) { + snapshotDisks.push({ disks: snapshot.disks, diskIndex }) } } - } catch (error) { - // no vmsd file :fall back to a full without snapshots + await asyncEach( + snapshotDisks, + async ({ disks: chainDisks, diskIndex }) => { + const disk = chainDisks[diskIndex] + chainDisks[diskIndex] = { + node: disk?.node, // 'scsi0:0' , 'ide0:0', ..., + ...(await inspect(disk.fileName)), + } + }, + { concurrency: DESCRIPTOR_CONCURRENCY, signal } + ) } + return { - name_label: config.name[0], - memory: +config.hardware[0].memoryMB[0] * 1024 * 1024, - nCpus: +config.hardware[0].numCPU[0], + name_label: config.name, + memory: +config.hardware.memoryMB * 1024 * 1024, + nCpus: +config.hardware.numCPU, guestToolsInstalled: false, - guestId: config.guestId[0], + guestId: config.guestId, guestFullName: config.guestFullName, - firmware: config.firmware[0] === 'efi' ? 'uefi' : config.firmware[0], // bios or uefi - powerState: runtime.powerState[0], + firmware: config.firmware === 'efi' ? 'uefi' : config.firmware, // bios or uefi + powerState: runtime.powerState, snapshots, cdrom, disks, @@ -417,37 +649,135 @@ export default class Esxi extends EventEmitter { } } - async #waitForTaskEnd(taskId) { - let state = 'running' - let info - for (let i = 0; i < 60; i++) { - // https://developer.vmware.com/apis/1720/ - info = await this.fetchProperty('Task', taskId, 'info') - state = info.state[0] - if (state === 'success') { - break - } - await new Promise(resolve => setTimeout(resolve, 1000)) + /** + * Extracts the id of the task started by a `*_Task` method. + * + * @param {object} result - result of the call + * @param {string} method + * @returns {string} + */ + #taskIdOf(result, method) { + const taskId = result?.returnval?.$value + if (taskId === undefined) { + const error = new Error(`${method} did not return a task`) + error.code = 'NO_TASK' + error.cause = result + throw error } - if (state === 'success') { - return info - } - warn('task not ended successfull ', { taskId, state, info }) - throw new Error('task execution failed') + return taskId } - async powerOff(vmId) { - const res = await this.#exec('PowerOffVM_Task', { _this: vmId }) - const taskId = res.returnval.$value + /** + * Polls a task until it ends. + * + * @param {string} taskId + * @param {object} [options] + * @param {string} [options.method] - name of the method which started the task, for the messages + * @param {number} [options.timeout] - in ms, how long the task is given to complete + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task + */ + async #waitForTaskEnd(taskId, { method = 'task', timeout = DEFAULT_TASK_TIMEOUT, signal } = {}) { + const start = Date.now() + let pollDelay = 500 + for (;;) { + signal?.throwIfAborted() + + let info + try { + // https://developer.vmware.com/apis/1720/ + info = await this.#retrieveProperty('Task', taskId, 'info', { signal }) + } catch (error) { + // the task keeps running on the host: a transient failure of a single poll must not fail + // an operation which legitimately lasts hours, and be reported as if nothing was running + if (!isRetryableError(error) || Date.now() - start >= timeout) { + throw error + } + warn('failed to read the state of a task, will poll again', { error, method, taskId }) + await delay(pollDelay, undefined, { signal }) + pollDelay = Math.min(pollDelay * 2, MAX_TASK_POLL_DELAY) + continue + } + + const { state } = info + + if (state === 'success') { + return info + } + + if (state === 'error') { + // don't burn the whole timeout on a task which already failed + const error = new Error(`${method} failed: ${taskFaultMessage(info.error) ?? 'unknown fault'}`) + // the fault type when the host named one, so that a caller can branch on it + error.code = taskFaultType(info.error) ?? 'TASK_FAILED' + error.cause = info.error + warn('task ended in error', { taskId, method, state, error }) + throw error + } + + if (Date.now() - start >= timeout) { + const error = new Error(`${method} did not complete within ${Math.round(timeout / 1000)}s (state: ${state})`) + // the task keeps running on the host: this is not retryable, starting a second one would + // not make the first go away + error.code = 'TASK_TIMEOUT' + error.cause = info + warn('task timed out', { taskId, method, state }) + throw error + } + + await delay(pollDelay, undefined, { signal }) + pollDelay = Math.min(pollDelay * 2, MAX_TASK_POLL_DELAY) + } + } + + /** + * Starts a vim25 method which returns a Task, and waits for that task to end. + * + * @param {string} method - name of the method, as exposed by the WSDL, e.g. `PowerOffVM_Task` + * @param {object} args - arguments of the method. `_this` names the VM the task runs on, and is + * attached to a failure. The schema declares them as a sequence, so they are sent in the order + * they are given in here + * @param {object} [options] + * @param {number} [options.timeout] - in ms, how long the task is given to complete + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task + */ + async #runTask(method, args, { signal, timeout } = {}) { + const vmId = args._this + const res = await this.#vimClient.call(method, args) try { - return await this.#waitForTaskEnd(taskId) + return await this.#waitForTaskEnd(this.#taskIdOf(res, method), { method, signal, timeout }) } catch (error) { - warn('Fail to power off VM', { vmId, error }) + error.vmId = vmId + warn('a task on a VM failed', { error, method, vmId }) throw error } } - powerOn(vmId) { - return this.#exec('PowerOnVM_Task', { _this: vmId }) + + /** + * Powers off a VM, and waits for it to be off. + * + * @param {string} vmId - id of the VM + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task + */ + async powerOff(vmId, { signal, timeout = 5 * 60e3 } = {}) { + return this.#runTask('PowerOffVM_Task', { _this: vmId }, { signal, timeout }) + } + + /** + * Powers on a VM, and waits for the hypervisor to have started it. + * + * @param {string} vmId - id of the VM + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task + */ + async powerOn(vmId, { signal, timeout = 5 * 60e3 } = {}) { + return this.#runTask('PowerOnVM_Task', { _this: vmId }, { signal, timeout }) } /** @@ -459,291 +789,408 @@ export default class Esxi extends EventEmitter { * identity of the disks of the chain. * * @param {string} vmId - id of the VM, must be powered on + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task */ - async reset(vmId) { - const res = await this.#exec('ResetVM_Task', { _this: vmId }) - const taskId = res.returnval.$value - try { - return await this.#waitForTaskEnd(taskId) - } catch (error) { - warn('Fail to reset VM', { vmId, error }) - throw error - } + async reset(vmId, { signal, timeout = 5 * 60e3 } = {}) { + return this.#runTask('ResetVM_Task', { _this: vmId }, { signal, timeout }) } /** * Removes every snapshot of a VM, consolidating their content into the base disks. * * @param {string} vmId - id of the VM + * @param {object} [options] + * @param {number} [options.timeout] - in ms, consolidating the deltas of a large disk takes as + * long as it takes + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task */ - async removeAllSnapshots(vmId) { - const res = await this.#exec('RemoveAllSnapshots_Task', { _this: vmId, consolidate: true }) - const taskId = res.returnval.$value - try { - return await this.#waitForTaskEnd(taskId) - } catch (error) { - warn('Fail to remove the snapshots of VM', { vmId, error }) - throw error - } - } - - async snapshot(vmId, name, description) { - const res = await this.#exec('CreateSnapshotEx_Task', { _this: vmId, name, description, memory: false }) - const taskId = res.returnval.$value - try { - return await this.#waitForTaskEnd(taskId) - } catch (error) { - warn('Fail to take a snapshot', { vmId, error }) - throw error - } - } - - async fetchProperty(type, id, propertyName) { - // the fetch method does not seems to be exposed by the wsdl - // inspired by the pyvmomi implementation ( StubAdapterAccessorImpl.py / InvokeAccessor) - const url = new URL('https://localhost/sdk') - url.host = this.#host - const res = await fetch(url, { - method: 'POST', - headers: { - Cookie: this.#client.authCookie.cookies, - SOAPAction: '"urn:vim25/6.0"', // mandatory to have an answer when asking for httpNfcLease - }, - dispatcher: this.#httpsAgent, - body: ` - - - - <_this type="${type}">${id} - ${propertyName} - - - `, - }) - const text = await res.text() - const matches = text.match(/]*>(.*)<\/FetchResponse>/s) - if (matches === null) { - throw new Error(`can't get ${propertyName} of object ${id} (Type: ${type})`) - } - return new Promise((resolve, reject) => { - xml2js.parseString(matches[1], (err, res) => (err ? reject(err) : resolve(res.returnval))) - }) + async removeAllSnapshots(vmId, { signal, timeout = 6 * 3600e3 } = {}) { + return this.#runTask('RemoveAllSnapshots_Task', { _this: vmId, consolidate: true }, { signal, timeout }) } /** - * get the thumbprint of the certificate on the esxi. Extracted from vddk-remote code + * Takes a snapshot of a VM, without its memory. + * + * @param {string} vmId - id of the VM + * @param {string} name + * @param {string} description + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @param {AbortSignal} [options.signal] + * @returns {Promise} the `info` of the successful task + */ + async snapshot(vmId, name, description, { signal, timeout = 30 * 60e3 } = {}) { + return this.#runTask( + 'CreateSnapshotEx_Task', + { _this: vmId, name, description, memory: false }, + { signal, timeout } + ) + } + + /** + * Reads one property of one managed object. + * + * Goes through the WSDL, and returns the value in the same shape as {@link search}. + * + * @param {string} type - type of the object, e.g. `VirtualMachine` + * @param {string} id - managed object reference of the object + * @param {string} path - property path, e.g. `config.hardware.device` + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @param {AbortSignal} [options.signal] + * @returns {Promise} + */ + async #retrieveProperty(type, id, path, { timeout, signal } = {}) { + signal?.throwIfAborted() + await this.#vimClient.connect() + const propertyCollector = this.#vimClient.serviceContent.propertyCollector + + const result = await this.#vimClient.call( + 'RetrievePropertiesEx', + { + _this: propertyCollector, + specSet: [ + propertyFilterSpec({ + propSet: [propertySpec({ type, pathSet: [path] })], + // the object itself, no container view to create and destroy + objectSet: [objectSpec({ obj: moRef(type, id), skip: false })], + }), + ], + options: retrieveOptions({}), + }, + { timeout } + ) + + const returnval = result?.returnval + if (returnval?.token !== undefined) { + // a single property of a single object always fits in one page, but a retrieval left open + // would keep its results on the server + await this.#vimClient + .call('CancelRetrievePropertiesEx', { _this: propertyCollector, token: returnval.token }) + .catch(error => warn('failed to cancel the property retrieval', { error, token: returnval.token })) + } + + const property = asArray(asArray(returnval?.objects)[0]?.propSet).find(({ name }) => name === path) + if (property === undefined) { + // the object may have been deleted since it was listed + const error = new Error(`can't get ${path} of object ${id} (Type: ${type})`) + error.code = 'NO_PROPERTY' + throw error + } + + return normalizeSoapValue(property.val) + } + + /** + * Touches the session on the host so that it does not expire. + * + * A vim25 session is dropped after 30 minutes without a call by default, and an import spends + * hours moving data outside of the WSDL. `CurrentTime` is the cheapest call which refreshes it: + * the keep-alive used to read the whole `config` of a VM instead. + * + * Only the session of the WSDL is concerned. The HTTP session used to download from a datastore + * carries its own cookies, and recovers on its own: a 401 drops them and the retry authenticates + * again. + * + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @returns {Promise} + */ + async keepAlive({ timeout } = {}) { + await this.#vimClient.call('CurrentTime', { _this: 'ServiceInstance' }, { timeout }) + } + + /** + * SHA-1 fingerprint of the certificate of the host, as the vddk library expects it. + * + * Memoized: it used to be computed again for every nbdkit server, with two openssl processes + * every time. + * * @returns {Promise} */ - async #getServerThumbprint() { - const tmpDir = await fs.mkdtemp(join(tmpdir(), 'xo-server')) - const certFile = join(tmpDir, 'cert') - - try { - const devnull = await fs.open('/dev/null') - // ensure arguments are properly escaped - const cert = await new Promise((resolve, reject) => { - const process = spawn('openssl', ['s_client', '-connect', `${this.#host}:443`]) - let cert = '' - let stderr = '' - devnull.createReadStream().pipe(process.stdin) - process.stdout.on('data', data => { - cert += data - }) - - process.stderr.on('data', data => { - stderr += data - }) - - process.on('close', code => { - if (code !== 0) { - reject(new Error(`cert got an error code ${code} ${stderr}`)) - } else { - resolve(cert) - } - }) - }) - await fs.writeFile(certFile, cert) - const sha = await new Promise((resolve, reject) => { - exec(`openssl x509 -in ${certFile} -fingerprint -sha1 -noout`, (err, stdout, stderr) => { - if (err) { - return reject(err) - } - if (stdout) { - const matches = stdout.match(/sha1 Fingerprint=([0-9A-F:]+)/i) - if (matches === null) { - throw new Error(`Can't extract server finger print`, { stdout, stderr, cert }) - } - return resolve(matches[1]) - } - reject(new Error(`no answer in handling server thumbprint `)) - }) - }) - return sha - } finally { - await fs.unlink(certFile).catch(() => {}) - } + async getServerThumbprint() { + return (this.#thumbprint ??= this.#computeServerThumbprint().catch(error => { + // a transient failure must not be memoized + this.#thumbprint = undefined + throw error + })) } - async spawnNbdKitProcess(vmId, diskPath, { singleLink = false, threads = 1, compression = 'fastlz' } = {}) { - const key = `${vmId}/${diskPath}/${singleLink}` - if (!this.#nbdServers.has(key)) { - const thumbprint = await this.#getServerThumbprint() - const port = nbdPort++ - const tmpDir = await fs.mkdtemp(join(tmpdir(), 'xo-server')) - const passFile = join(tmpDir, 'params') - const outFd = await fs.open(join(tmpDir, 'stdout'), 'a') - const outFile = outFd.createWriteStream() - const errFd = await fs.open(join(tmpDir, 'stderr'), 'a') - const errFile = errFd.createWriteStream() - await fs.writeFile(passFile, this.#password) - const args = [ - '-r', // readonly - '-v', - '-f', - '--exit-with-parent', // implies -f , ensure we don't leave orphans - `--threads=${threads}`, - `--port=${port}`, - 'vddk', // the vddk plugin - `compression=${compression}`, - `thumbprint=${thumbprint}`, - `server=${this.#host}`, - `user=${this.#user}`, - `password=+${passFile}`, - `libdir=${VDDK_LIB_PATH}`, - `vm=moref=${vmId}`, - singleLink ? 'single-link=true' : '', - diskPath, - ] - try { - const nbdKitProcess = spawn('nbdkit', args, { - cwd: tmpDir, - env: { - ...process.env, - LD_LIBRARY_PATH: `${VDDK_LIB_PATH}/lib64`, - }, - }) - nbdKitProcess.stdout.pipe(outFile) - nbdKitProcess.stderr.pipe(errFile) - this.#nbdServers.set(key, { - process: nbdKitProcess, - nbdInfos: { address: '127.0.0.1', port, exportname: diskPath }, - }) + #computeServerThumbprint() { + // the host may carry a port, e.g. `esxi.example:8443`, since it is used as the host of an url + const { hostname, port } = new URL(`https://${this.#host}`) + return getCertificateThumbprint(hostname.replace(/^\[|\]$/g, ''), port === '' ? {} : { port: Number(port) }) + } - info(`nbdkit logs of ${diskPath} are in ${tmpDir}`) + /** + * Starts an nbdkit server exporting a disk of a VM, or shares the one already serving it. + * + * The server is stopped when the disposable is disposed, and a server shared by several callers + * survives until the last of them disposes: the caller no longer has to name it again, with the + * exact same options, to stop it. + * + * @param {string} vmId + * @param {string} diskPath - `[datastore] dir/disk.vmdk` + * @param {object} [options] + * @param {string} [options.compression] + * @param {boolean} [options.singleLink] - export the top delta only + * @param {number} [options.threads] + * @returns {Promise, nbdInfos: object, process: object }>>} + */ + async getNbdServer(vmId, diskPath, { compression = 'fastlz', singleLink = false, threads = 1 } = {}) { + const key = nbdServerKey(vmId, diskPath, { compression, singleLink, threads }) - nbdKitProcess.on('close', code => { - if (code !== 0) { - warn(`nbdkit server process exited with code ${code} ,detailed logs are in ${tmpDir}/stderr `) - } - }) - // @todo find a better to wait for server ready - await new Promise(resolve => setTimeout(resolve, 2000)) - } finally { - fs.unlink(passFile).catch(warn) + let entry = this.#nbdServers.get(key) + if (entry === undefined) { + // the promise is memoized, not its result: the previous implementation had six await points + // between the check and the registration, so two concurrent calls spawned two servers and + // orphaned the first one + entry = { pending: this.#spawnNbdKitProcess(vmId, diskPath, { compression, singleLink, threads }), users: 0 } + this.#nbdServers.set(key, entry) + + // neither a failed spawn nor a dead server must be handed out to the next caller + const forget = () => { + if (this.#nbdServers.get(key) === entry) { + this.#nbdServers.delete(key) + } + } + entry.pending.then(server => server.died.then(forget), forget) + } + + entry.users += 1 + + let server + try { + server = await entry.pending + } catch (error) { + entry.users -= 1 + throw error + } + + // `Disposable` refuses a second disposal itself, so this runs exactly once per caller + const dispose = async () => { + entry.users -= 1 + // `close()` stops every server left, a disposal after it has nothing to do + if (entry.users === 0 && this.#nbdServers.get(key) === entry) { + await this.#killNbdServerByKey(key) } } - return this.#nbdServers.get(key) + + return new Disposable(dispose, server) } - async killNbdServer(vmId, diskPath, { singleLink = false } = {}) { - const key = `${vmId}/${diskPath}/${singleLink}` - if (!this.#nbdServers.has(key)) { - warn(` process ${vmId}/${diskPath}/${singleLink} was already killed`) - } else { - this.#nbdServers.get(key).process.kill() + + async #spawnNbdKitProcess(vmId, diskPath, { compression, singleLink, threads }) { + const thumbprint = await this.getServerThumbprint() + const port = await findFreePort() + const tmpDir = await fs.mkdtemp(join(tmpdir(), 'xo-server')) + const passFile = join(tmpDir, 'params') + const outFd = await fs.open(join(tmpDir, 'stdout'), 'a') + const errFd = await fs.open(join(tmpDir, 'stderr'), 'a') + // the file holds a password, and only the directory was protecting it + await fs.writeFile(passFile, this.#password, { mode: 0o600 }) + + const args = formatNbdkitArgs({ + compression, + diskPath, + host: this.#host, + libdir: VDDK_LIB_PATH, + passFile, + port, + singleLink, + threads, + thumbprint, + user: this.#user, + vmId, + }) + + const nbdKitProcess = this.#spawn('nbdkit', args, { + cwd: tmpDir, + env: { + ...process.env, + LD_LIBRARY_PATH: `${VDDK_LIB_PATH}/lib64`, + }, + }) + nbdKitProcess.stdout.pipe(outFd.createWriteStream()) + nbdKitProcess.stderr.pipe(errFd.createWriteStream()) + info(`nbdkit logs of ${diskPath} are in ${tmpDir}`) + + // `error` is emitted when the binary is missing: without a listener, it is an uncaught event + // which terminates the whole process + const died = new Promise(resolve => { + nbdKitProcess.once('error', error => resolve({ error })) + nbdKitProcess.once('exit', (code, signal) => resolve({ code, signal })) + }) + + died.then(async ({ code, error, signal }) => { + await Promise.all([outFd.close().catch(noop), errFd.close().catch(noop)]) + if (error !== undefined) { + warn('nbdkit could not be started', { args, error, tmpDir }) + } else if (code !== 0) { + warn(`nbdkit server process exited with code ${code} ,detailed logs are in ${tmpDir}/stderr `, { signal }) + } else { + // nothing to look at, the logs would pile up in the temporary directory + await fs.rm(tmpDir, { force: true, recursive: true }).catch(noop) + } + }) + + // the readiness of the server and its death are racing: nbdkit exits on a bad thumbprint or a + // missing library, and waiting for the port would then burn the whole timeout + const failed = died.then(({ code, error }) => { + if (error !== undefined) { + // the spawn itself failed, and its own code already says why, e.g. `ENOENT` when nbdkit is + // not installed + throw error + } + const exited = new Error( + `nbdkit exited with code ${code} before being ready, detailed logs are in ${tmpDir}/stderr` + ) + exited.code = 'NBDKIT_EXITED' + throw exited + }) + failed.catch(noop) // the race is usually won by the readiness of the server + + const readiness = new AbortController() + try { + await Promise.race([waitForPort(port, { signal: readiness.signal, timeout: NBDKIT_READY_TIMEOUT }), failed]) + } catch (error) { + await this.#killNbdServer({ died, process: nbdKitProcess }, diskPath).catch(noop) + throw error + } finally { + // losing the race must not leave a probe running until its own timeout + readiness.abort() + // nbdkit reads the password once, while configuring its plugin, which is done by the time it + // listens + await fs.unlink(passFile).catch(error => warn('failed to remove the password file', { error, passFile })) } + + return { + died, + nbdInfos: { address: '127.0.0.1', port, exportname: diskPath }, + process: nbdKitProcess, + } + } + + async #killNbdServer(server, label) { + const { died, process: nbdKitProcess } = server + if (nbdKitProcess.exitCode !== null || nbdKitProcess.signalCode !== null) { + return + } + nbdKitProcess.kill() + try { + await pTimeout.call(died, NBDKIT_KILL_TIMEOUT) + } catch (error) { + warn('nbdkit did not exit, killing it', { error, label, pid: nbdKitProcess.pid }) + nbdKitProcess.kill('SIGKILL') + await died + } + } + + async #killNbdServerByKey(key) { + const entry = this.#nbdServers.get(key) + if (entry === undefined) { + warn(`nbdkit server ${key} was already killed`) + return + } + // the entry used to be left in place, so the next spawn handed out a dead process listening on + // nothing + this.#nbdServers.delete(key) + + let server + try { + server = await entry.pending + } catch { + // the spawn failed, there is nothing left to kill + return + } + await this.#killNbdServer(server, key) } async #getDataMapFromVddk(vmId, datastoreName, diskPath, signal) { - let nbdClient - try { - const start = Date.now() + return Disposable.use( + this.getNbdServer(vmId, `[${datastoreName}] ${diskPath}`, { singleLink: true }), + async ({ nbdInfos }) => { + const start = Date.now() + info('nbd server for data map spawned') + signal?.throwIfAborted() - const nbdInfoSpawn = await this.spawnNbdKitProcess(vmId, `[${datastoreName}] ${diskPath}`, { - singleLink: true, - }) + // the client is built here, and not before the server: a spawn which failed used to leave + // it undefined, and the `TypeError` of `nbdClient.disconnect()` replaced the real error, + // feeding the fallback with a misleading cause + const nbdClient = new NbdClient(nbdInfos) + try { + await nbdClient.connect() - info(`nbd server for data map spawned`) - signal?.throwIfAborted() + info('nbd client for data map connected') - nbdClient = new NbdClient(nbdInfoSpawn.nbdInfos) + const dataMap = await nbdClient.getMap(signal) - await nbdClient.connect() + info( + `got the data map of the single disk in ${Math.round((Date.now() - start) / 1000)} seconds ,${dataMap.length} blocks` + ) - info(`nbd client for data map connected`) + return dataMap + } finally { + await nbdClient.disconnect().catch(error => warn('error while disconnecting the nbd client', { error })) + } + } + ) + } - const dataMap = await nbdClient.getMap(signal) - - info( - `got the data map of the single disk in ${Math.round((Date.now() - start) / 1000)} seconds ,${dataMap.length} blocks` - ) - - return dataMap - } finally { - await nbdClient.disconnect() - await this.killNbdServer(vmId, `[${datastoreName}] ${diskPath}`, { singleLink: true }).catch(err => - warn('error while stopping nbdkit server for the snapshot', err) - ) - } + async #readRange(datastoreName, path, start, length, signal) { + // an HTTP range is inclusive: asking for `0-512` reads 513 bytes + const res = await this.download(datastoreName, path, { range: `${start}-${start + length - 1}`, signal }) + return Buffer.from(await res.arrayBuffer()) } async #getDataMapFromCowd(datastoreName, diskPath, signal) { - const descriptorResponse = await this.download(datastoreName, diskPath, '0-512', signal) - const descriptorBlob = await new Response(descriptorResponse.body).blob() - const descriptorBytes = new Uint8Array(await descriptorBlob.arrayBuffer()).slice(0, 512) - - const parsedDescriptor = parseVmdk(new TextDecoder('utf-8').decode(descriptorBytes)) - - const diskPathArray = diskPath.split('/') - const extentPath = diskPathArray.slice(0, -1).join('/') + '/' + parsedDescriptor.fileName - - const extentHeaderResponse = await this.download(datastoreName, extentPath, `0-2048`, signal) - const extentHeaderBlob = await new Response(extentHeaderResponse.body).blob() - const extentHeaderBuffer = Buffer.from(await extentHeaderBlob.arrayBuffer()) - - strictEqual(extentHeaderBuffer.subarray(0, 4).toString('ascii'), 'COWD') - - const extentNumGdEntries = extentHeaderBuffer.readUInt32LE(24) - - const extentGDResponse = await this.download( - datastoreName, - extentPath, - `2048-${2048 + extentNumGdEntries * 4}`, - signal - ) - const extentGDBlob = await new Response(extentGDResponse.body).blob() - const extentGDBuffer = Buffer.from(await extentGDBlob.arrayBuffer()) - - const dataMap = [] - let offset = 0 - for (let i = 0; i < extentNumGdEntries; i++) { - const extentGDE = extentGDBuffer.readUInt32LE(i * 4) - if (extentGDE !== 0) { - dataMap.push({ - offset, - length: 4096 * 512, - type: 0, - }) - } - - // Number of grains in a grain table * size of a grain. - offset += 4096 * 512 + const descriptor = await this.#readRange(datastoreName, diskPath, 0, DESCRIPTOR_READ_LENGTH, signal) + let fileName + try { + fileName = parseVmdk(descriptor.toString('utf8')).fileName + } catch (error) { + // this is the fallback of a failure: an assertion error here would hide the real problem + const wrapped = new Error(`can't read the descriptor of ${diskPath}`) + wrapped.code = 'NO_DATA_MAP' + wrapped.cause = error + throw wrapped } - return dataMap + const extentPath = diskPath.split('/').slice(0, -1).concat(fileName).join('/') + + const geometry = parseCowdHeader(await this.#readRange(datastoreName, extentPath, 0, COWD_HEADER_LENGTH, signal)) + const { grainDirectoryOffset, numGdEntries } = geometry + + const grainDirectory = await this.#readRange( + datastoreName, + extentPath, + grainDirectoryOffset, + numGdEntries * 4, + signal + ) + + return grainDirectoryToDataMap(grainDirectory, geometry) } + /** + * Blocks of a disk which hold data, used to transfer a delta instead of the whole disk. + * + * @param {string} vmId + * @param {string} datastoreName + * @param {string} diskPath - path of the disk in its datastore + * @param {AbortSignal} [signal] + * @returns {Promise>} + */ async getDataMap(vmId, datastoreName, diskPath, signal) { try { // We await the result of getDataMapFromVddk so we can catch errors and fallback to the direct metadata reading. return await this.#getDataMapFromVddk(vmId, datastoreName, diskPath, signal) } catch (error) { signal?.throwIfAborted() - warn('error while getting datamap from vddk, fall back to a direct metadata reading', error) + warn('error while getting datamap from vddk, fall back to a direct metadata reading', { error }) return this.#getDataMapFromCowd(datastoreName, diskPath, signal) } } diff --git a/@xen-orchestra/vmware-explorer/esxi.test.mjs b/@xen-orchestra/vmware-explorer/esxi.test.mjs new file mode 100644 index 0000000000..17f0441921 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/esxi.test.mjs @@ -0,0 +1,859 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { once } from 'node:events' + +import { connectedEsxi, DATACENTERS, FakeVimClient, moRef, page, response } from './esxi.fixtures.mjs' +import Esxi from './esxi.mjs' + +describe('search', function () { + it('destroys the container view it created', async function () { + const { esxi, vimClient } = await connectedEsxi({ responses: { RetrievePropertiesEx: () => page([]) } }) + + await esxi.search('VirtualMachine', ['config']) + + const destroyed = vimClient.callsTo('DestroyView') + assert.equal(destroyed.length, 1) + assert.deepEqual(destroyed[0].args, { _this: moRef('ContainerView', 'session[42]view-1') }) + }) + + it('returns an empty result instead of throwing on an empty inventory', async function () { + // no `returnval` at all + const { esxi, vimClient } = await connectedEsxi({ responses: { RetrievePropertiesEx: () => ({}) } }) + + assert.deepEqual(await esxi.search('VirtualMachine', ['config']), {}) + assert.equal(vimClient.callsTo('DestroyView').length, 1) + }) + + it('keeps falsy scalars', async function () { + const { esxi } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page([ + { + obj: moRef('VirtualMachine', 'vm-1'), + propSet: [ + { name: 'empty', val: { $value: '', attributes: { 'xsi:type': 'string' } } }, + { name: 'zero', val: { $value: 0 } }, + { name: 'no', val: { $value: false } }, + ], + }, + ]), + }, + }) + + assert.deepEqual(await esxi.search('VirtualMachine', ['empty', 'zero', 'no']), { + 'vm-1': { empty: '', zero: 0, no: false }, + }) + }) + + it('handles an object without any readable property', async function () { + const { esxi } = await connectedEsxi({ + responses: { RetrievePropertiesEx: () => page({ obj: moRef('VirtualMachine', 'vm-1') }) }, + }) + + assert.deepEqual(await esxi.search('VirtualMachine', ['config']), { 'vm-1': {} }) + }) + + it('merges the pages of an object split across them', async function () { + let continued = 0 + const { esxi, vimClient } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page( + [{ obj: moRef('VirtualMachine', 'vm-1'), propSet: [{ name: 'name', val: { $value: 'vm' } }] }], + 'token-1' + ), + ContinueRetrievePropertiesEx: () => + ++continued === 1 + ? page( + [ + { + obj: moRef('VirtualMachine', 'vm-1'), + propSet: [{ name: 'runtime', val: { attributes: {}, powerState: 'poweredOn' } }], + }, + ], + 'token-1' + ) + : page([{ obj: moRef('VirtualMachine', 'vm-2'), propSet: [{ name: 'name', val: { $value: 'other' } }] }]), + }, + }) + + assert.deepEqual(await esxi.search('VirtualMachine', ['name', 'runtime']), { + 'vm-1': { name: 'vm', runtime: { powerState: 'poweredOn' } }, + 'vm-2': { name: 'other' }, + }) + // the retrieval was consumed entirely, there is nothing to cancel + assert.equal(vimClient.callsTo('CancelRetrievePropertiesEx').length, 0) + assert.equal(vimClient.callsTo('DestroyView').length, 1) + }) + + it('cancels an interrupted retrieval and still destroys the view', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => page([], 'token-1'), + ContinueRetrievePropertiesEx: () => { + throw new Error('connection reset') + }, + }, + }) + + await assert.rejects(esxi.search('VirtualMachine', ['config']), /connection reset/) + + assert.deepEqual(vimClient.callsTo('CancelRetrievePropertiesEx')[0].args, { + _this: moRef('PropertyCollector', 'propertyCollector'), + token: 'token-1', + }) + assert.equal(vimClient.callsTo('DestroyView').length, 1) + }) + + it('asks for a bounded page size', async function () { + const { esxi, vimClient } = await connectedEsxi({ responses: { RetrievePropertiesEx: () => page([]) } }) + + await esxi.search('VirtualMachine', ['config'], { maxObjects: 10 }) + + assert.deepEqual(vimClient.callsTo('RetrievePropertiesEx')[0].args.options, { + attributes: { 'xsi:type': 'RetrieveOptions' }, + maxObjects: 10, + }) + }) +}) + +describe('connection', function () { + it('emits ready once the datacenters are known, without one request per datastore', async function () { + const vimClient = new FakeVimClient() + const esxi = new Esxi('esxi.test', 'user', 'password', true, { vimClient }) + + await once(esxi, 'ready') + + assert.deepEqual(vimClient.methods.toSorted(), [ + 'CreateContainerView', + 'CreateContainerView', + 'DestroyView', + 'DestroyView', + 'RetrievePropertiesEx', + 'RetrievePropertiesEx', + ]) + }) + + it('resolves the datacenter of a datastore when downloading', async function () { + const requests = [] + const { esxi } = await connectedEsxi({ + fetch: async (url, options) => { + requests.push({ url, options }) + return response({ status: 206 }) + }, + }) + + await esxi.download('ds main', 'vm/vm.vmdk', { range: '0-511' }) + + const { url, options } = requests[0] + assert.equal(url.host, 'esxi.test') + assert.equal(url.pathname, '/folder/vm/vm.vmdk') + assert.equal(url.searchParams.get('dsName'), 'ds main') + assert.equal(url.searchParams.get('dcPath'), 'dc-main') + assert.equal(options.headers.Range, 'bytes=0-511') + // a request body has no content type + assert.equal(options.headers['content-type'], undefined) + }) + + it('maps the datastores of every datacenter', async function () { + const requested = [] + const { esxi } = await connectedEsxi({ + fetch: async url => { + requested.push(url.searchParams.get('dcPath')) + return response() + }, + }) + + await esxi.download('ds2', 'a.vmdk') + await esxi.download('ds3', 'a.vmdk') + + assert.deepEqual(requested, ['dc-main', 'dc-other']) + }) + + it('closes the session', async function () { + const { esxi, vimClient } = await connectedEsxi() + + await esxi.close() + + assert.equal(vimClient.closed, true) + }) +}) + +describe('download', function () { + const downloadEsxi = async fetchImplementation => { + const requests = [] + const { esxi } = await connectedEsxi({ + fetch: async (url, options) => { + requests.push({ url, options }) + return fetchImplementation(requests.length, options) + }, + }) + return { esxi, requests } + } + + it('reuses the session cookie instead of authenticating on every request', async function () { + const { esxi, requests } = await downloadEsxi(() => + response({ headers: { 'set-cookie': 'vmware_soap_session="42"; Path=/; HttpOnly' } }) + ) + + await esxi.download('ds2', 'a.vmdk') + await esxi.download('ds2', 'a.vmdk') + + assert.equal(requests[0].options.headers.Authorization?.startsWith('Basic '), true) + assert.equal(requests[1].options.headers.Authorization, undefined) + assert.equal(requests[1].options.headers.cookie, 'vmware_soap_session="42"') + }) + + it('authenticates again when the session expired', async function () { + const { esxi, requests } = await downloadEsxi((attempt, options) => { + if (attempt === 1) { + return response({ headers: { 'set-cookie': 'vmware_soap_session="42"' } }) + } + // the host rejects the cookie of an expired session + return options.headers.cookie !== undefined + ? response({ status: 401, statusText: 'Unauthorized' }) + : response({ headers: { 'set-cookie': 'vmware_soap_session="1337"' } }) + }) + + await esxi.download('ds2', 'a.vmdk') + await esxi.download('ds2', 'a.vmdk', { retryDelay: 1 }) + + assert.equal(requests.length, 3) + assert.equal(requests[1].options.headers.cookie, 'vmware_soap_session="42"') + // the retry falls back to the credentials, and picks up the new session + assert.equal(requests[2].options.headers.Authorization?.startsWith('Basic '), true) + }) + + it('does not retry a missing file', async function () { + const { esxi, requests } = await downloadEsxi(() => + response({ status: 404, statusText: 'Not Found', body: 'no such file' }) + ) + + await assert.rejects(esxi.download('ds2', 'missing.vmdk'), error => { + assert.match(error.message, /^404 Not Found /) + assert.equal(error.cause.status, 404) + assert.equal(error.cause.body, 'no such file') + return true + }) + assert.equal(requests.length, 1) + }) + + it('does not retry an unknown datastore', async function () { + const { esxi, requests } = await downloadEsxi(() => response()) + + await assert.rejects(esxi.download('not-a-datastore', 'a.vmdk')) + + assert.equal(requests.length, 0) + }) + + it('retries a network failure', async function () { + const { esxi, requests } = await downloadEsxi(attempt => { + if (attempt < 3) { + const error = new TypeError('fetch failed') + error.cause = { code: 'ECONNRESET' } + throw error + } + return response() + }) + + await esxi.download('ds2', 'a.vmdk', { retryDelay: 1 }) + + assert.equal(requests.length, 3) + }) + + it('gives up after the last retry', async function () { + const { esxi, requests } = await downloadEsxi(() => response({ status: 503, statusText: 'Service Unavailable' })) + + await assert.rejects(esxi.download('ds2', 'a.vmdk', { retries: 2, retryDelay: 1 }), { message: /^503 / }) + + assert.equal(requests.length, 3) + }) + + it('stops retrying when aborted', async function () { + const controller = new AbortController() + const { esxi, requests } = await downloadEsxi(() => { + controller.abort() + const error = new Error('socket hang up') + error.code = 'ECONNRESET' + throw error + }) + + await assert.rejects(esxi.download('ds2', 'a.vmdk', { retryDelay: 1, signal: controller.signal }), { + name: 'AbortError', + }) + assert.equal(requests.length, 1) + }) + + it('fails when the host answers more than the requested range', async function () { + // reading the answer as if it were the range would use as much memory as the file is big + const { esxi, requests } = await downloadEsxi(() => response({ status: 200, body: Buffer.alloc(4096) })) + + await assert.rejects(esxi.download('ds2', 'a.vmdk', { range: '0-511', retryDelay: 1 }), { + code: 'RANGE_IGNORED', + message: /^the range 0-511 was ignored by the host \(status 200, 4096 bytes\)/, + }) + assert.equal(requests.length, 1) + }) + + it('fails when the size of the answer to a range is unknown', async function () { + // a chunked answer cannot be checked, and `Number(null)` is 0: the whole file used to pass + const { esxi } = await downloadEsxi(() => ({ + status: 200, + statusText: 'OK', + headers: new Headers(), + body: { cancel: async () => {} }, + arrayBuffer: async () => new ArrayBuffer(4096), + })) + + await assert.rejects(esxi.download('ds2', 'huge.vmdk', { range: '0-2047', retryDelay: 1 }), { + code: 'RANGE_IGNORED', + message: /^the range 0-2047 was ignored by the host \(status 200, null bytes\)/, + }) + }) + + it('accepts a whole file answered to a range which covers it', async function () { + // what `/folder` does for a vmdk descriptor of a few hundred bytes read as `0-511` + const { esxi } = await downloadEsxi(() => response({ status: 200, body: Buffer.alloc(293) })) + + const res = await esxi.download('ds2', 'descriptor.vmdk', { range: '0-511' }) + + assert.equal((await res.arrayBuffer()).byteLength, 293) + }) + + it('fails when a range not starting at the first byte is answered with a 200', async function () { + // such an answer starts at the first byte, using it would read the wrong offset + const { esxi } = await downloadEsxi(() => response({ status: 200, body: Buffer.alloc(16) })) + + await assert.rejects(esxi.download('ds2', 'a.vmdk', { range: '2048-2063', retryDelay: 1 }), { + code: 'RANGE_IGNORED', + message: /^the range 2048-2063 was ignored by the host/, + }) + }) +}) + +// a RetrievePropertiesEx response for a single object and a single property +const propertyOf = (type, id, name, val) => ({ + returnval: { objects: [{ obj: moRef(type, id), propSet: [{ name, val }] }] }, +}) + +const taskInfo = (state, extra = {}) => + propertyOf('Task', 'task-1', 'info', { + attributes: { 'xsi:type': 'TaskInfo' }, + key: 'task-1', + state, + ...extra, + }) + +const startedTask = () => ({ returnval: moRef('Task', 'task-1') }) + +describe('tasks', function () { + it('resolves with the info of the successful task', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + RetrievePropertiesEx: () => taskInfo('success'), + }, + }) + + // the info comes back unwrapped, in the same shape as `search` + assert.deepEqual(await esxi.powerOff('vm-1'), { key: 'task-1', state: 'success' }) + + // the property is read on the task itself, no container view is involved + assert.deepEqual(vimClient.methods, ['PowerOffVM_Task', 'RetrievePropertiesEx']) + const { specSet } = vimClient.callsTo('RetrievePropertiesEx')[0].args + assert.deepEqual(specSet[0].objectSet[0].obj, { attributes: { type: 'Task' }, $value: 'task-1' }) + assert.deepEqual(specSet[0].propSet[0].pathSet, ['info']) + }) + + it('fails as soon as the task is in error, with the fault of the host', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + RetrievePropertiesEx: () => + taskInfo('error', { + error: { + attributes: { 'xsi:type': 'LocalizedMethodFault' }, + fault: { attributes: { 'xsi:type': 'InvalidPowerState' } }, + localizedMessage: 'The attempted operation cannot be performed in the current state (Powered off).', + }, + }), + }, + }) + + await assert.rejects(esxi.powerOff('vm-1'), error => { + assert.match(error.message, /^PowerOffVM_Task failed: The attempted operation cannot be performed/) + assert.equal(error.code, 'InvalidPowerState') + assert.equal(error.vmId, 'vm-1') + return true + }) + + // a failed task is not polled until the timeout + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 1) + }) + + it('falls back to a code of its own when the host names no fault type', async function () { + const { esxi } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + RetrievePropertiesEx: () => taskInfo('error', { error: { localizedMessage: 'it did not work' } }), + }, + }) + + await assert.rejects(esxi.powerOff('vm-1'), { + code: 'TASK_FAILED', + message: 'PowerOffVM_Task failed: it did not work', + }) + }) + + it('gives up when the task takes longer than its timeout', async function () { + const { esxi } = await connectedEsxi({ + responses: { + RemoveAllSnapshots_Task: () => startedTask(), + RetrievePropertiesEx: () => taskInfo('running'), + }, + }) + + await assert.rejects(esxi.removeAllSnapshots('vm-1', { timeout: 0 }), { + code: 'TASK_TIMEOUT', + message: 'RemoveAllSnapshots_Task did not complete within 0s (state: running)', + }) + }) + + it('stops polling when aborted', async function () { + const controller = new AbortController() + const { esxi, vimClient } = await connectedEsxi({ + responses: { + CreateSnapshotEx_Task: () => startedTask(), + RetrievePropertiesEx: () => { + controller.abort() + return taskInfo('running') + }, + }, + }) + + await assert.rejects(esxi.snapshot('vm-1', 'name', 'description', { signal: controller.signal }), { + name: 'AbortError', + }) + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 1) + }) + + it('waits for the completion of a power on', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { + PowerOnVM_Task: () => startedTask(), + RetrievePropertiesEx: () => taskInfo('success'), + }, + }) + + await esxi.powerOn('vm-1') + + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 1) + }) + + it('polls again while the task is running', async function () { + let polls = 0 + const { esxi, vimClient } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + RetrievePropertiesEx: () => taskInfo(++polls === 1 ? 'running' : 'success'), + }, + }) + + await esxi.powerOff('vm-1') + + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 2) + }) + + it('keeps polling a task through a transient failure', async function () { + let polls = 0 + const { esxi, vimClient } = await connectedEsxi({ + responses: { + RemoveAllSnapshots_Task: () => startedTask(), + RetrievePropertiesEx: () => { + if (++polls === 1) { + // the task keeps running on the host, reporting a failure would be a lie + const error = new Error('socket hang up') + error.code = 'ECONNRESET' + throw error + } + return taskInfo('success') + }, + }, + }) + + await esxi.removeAllSnapshots('vm-1') + + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 2) + }) + + it('does not retry a poll which will not succeed', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + // the task object is gone + RetrievePropertiesEx: () => ({}), + }, + }) + + await assert.rejects(esxi.powerOff('vm-1'), { code: 'NO_PROPERTY' }) + + assert.equal(vimClient.callsTo('RetrievePropertiesEx').length, 1) + }) + + it('reports a method which did not start a task', async function () { + const { esxi } = await connectedEsxi({ responses: { ResetVM_Task: () => ({}) } }) + + await assert.rejects(esxi.reset('vm-1'), { + code: 'NO_TASK', + message: 'ResetVM_Task did not return a task', + }) + }) + + it('reports a task which disappeared', async function () { + const { esxi } = await connectedEsxi({ + responses: { + PowerOffVM_Task: () => startedTask(), + // the object is gone: no property is returned + RetrievePropertiesEx: () => ({}), + }, + }) + + await assert.rejects(esxi.powerOff('vm-1'), { code: 'NO_PROPERTY' }) + }) +}) + +describe('getAllVmMetadata', function () { + const vm = (id, propSet) => ({ obj: moRef('VirtualMachine', id), propSet }) + + const CONFIG = { + name: 'config', + val: { attributes: {}, name: 'a vm', firmware: 'efi', hardware: { memoryMB: '2048', numCPU: '2' } }, + } + const STORAGE = { + name: 'storage', + val: { attributes: {}, perDatastoreUsage: { committed: '1024', uncommitted: '2048' } }, + } + const RUNTIME = { name: 'runtime', val: { attributes: {}, powerState: 'poweredOn' } } + + it('reports the VMs of the host', async function () { + const { esxi } = await connectedEsxi({ + responses: { RetrievePropertiesEx: () => page([vm('vm-1', [CONFIG, STORAGE, RUNTIME])]) }, + }) + + assert.deepEqual(await esxi.getAllVmMetadata(), [ + { + id: 'vm-1', + hasAllExtentsListed: true, + nameLabel: 'a vm', + memory: 2048 * 1024 * 1024, + nCpus: 2, + guestToolsInstalled: false, + firmware: 'uefi', + powerState: 'poweredOn', + storage: { used: 1024, free: 2048 }, + }, + ]) + }) + + it('ignores a VM whose properties are incomplete instead of failing the listing', async function () { + const { esxi } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page([ + // a VM being created has no readable runtime + vm('vm-1', [CONFIG, STORAGE]), + vm('vm-2', [CONFIG, STORAGE, RUNTIME]), + ]), + }, + }) + + assert.deepEqual( + (await esxi.getAllVmMetadata()).map(({ id }) => id), + ['vm-2'] + ) + }) + + it('ignores a VM whose config is reported without its hardware', async function () { + // a VM being created: `config` is there, `config.hardware` is not yet + const { esxi } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page([ + vm('vm-1', [{ name: 'config', val: { attributes: {}, name: 'new vm' } }, STORAGE, RUNTIME]), + vm('vm-2', [CONFIG, STORAGE, RUNTIME]), + ]), + }, + }) + + assert.deepEqual( + (await esxi.getAllVmMetadata()).map(({ id }) => id), + ['vm-2'] + ) + }) + + it('ignores a VM whose runtime is reported without its power state', async function () { + const { esxi } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page([vm('vm-1', [CONFIG, STORAGE, { name: 'runtime', val: { attributes: {}, host: 'host-1' } }])]), + }, + }) + + assert.deepEqual(await esxi.getAllVmMetadata(), []) + }) + + it('detects a disk whose extent is not listed', async function () { + const layoutEx = files => ({ name: 'layoutEx', val: { attributes: {}, ...files } }) + const { esxi } = await connectedEsxi({ + responses: { + RetrievePropertiesEx: () => + page([ + vm('vm-1', [ + CONFIG, + STORAGE, + RUNTIME, + // a single chain and a single file are not wrapped in arrays + layoutEx({ + disk: { chain: { fileKey: ['1', '2'] } }, + file: { key: '2', type: 'diskDescriptor' }, + }), + ]), + ]), + }, + }) + + assert.equal((await esxi.getAllVmMetadata())[0].hasAllExtentsListed, false) + }) +}) + +describe('getTransferableVmMetadata', function () { + const VMX = `config.version = "8" +scsi0.present = "TRUE" +scsi0.virtualDev = "lsilogic" +scsi0:1.present = "TRUE" +scsi0:1.deviceType = "scsi-hardDisk" +scsi0:1.fileName = "vm-000001.vmdk" +ide0:0.present = "TRUE" +ide0:0.deviceType = "cdrom-image" +ethernet0.present = "TRUE" +ethernet0.networkName = "VM Network" +ethernet0.addressType = "generated" +ethernet0.generatedAddress = "00:0c:29:00:00:01" +` + + const DESCRIPTOR = `# Disk DescriptorFile +version=1 +CID=d7980f7a +parentCID=ffffffff +createType="vmfs" + +RW 67108864 VMFS "vm-000001-flat.vmdk" +` + + const config = vmPathName => ({ + attributes: { 'xsi:type': 'VirtualMachineConfigInfo' }, + name: 'a vm', + guestId: 'ubuntu64Guest', + guestFullName: 'Ubuntu Linux (64-bit)', + firmware: 'efi', + files: { vmPathName }, + hardware: { memoryMB: '2048', numCPU: '2' }, + }) + + const RUNTIME = { attributes: { 'xsi:type': 'VirtualMachineRuntimeInfo' }, powerState: 'poweredOn' } + + const DATASTORE_SUMMARIES = [ + { + obj: moRef('Datastore', 'datastore-11'), + propSet: [{ name: 'summary', val: { attributes: {}, name: 'ds main', url: '/vmfs/volumes/uuid-1' } }], + }, + ] + + /** + * @param {object} [options] + * @param {string} [options.vmPathName] - `files.vmPathName` reported for the VM + * @param {number} [options.vmsdStatus] - status of the response to the vmsd request + */ + const transferableEsxi = async ({ vmPathName = '[ds main] a.vm/a.vm.vmx', vmsdStatus = 404 } = {}) => { + const requested = [] + const { esxi } = await connectedEsxi({ + responses: { + // every property is read through the property collector, only the files are downloaded + RetrievePropertiesEx: ({ specSet }) => { + const { pathSet, type } = specSet[0].propSet[0] + if (type === 'Datastore') { + return page(DATASTORE_SUMMARIES) + } + if (type !== 'VirtualMachine') { + return page(DATACENTERS) + } + return pathSet[0] === 'runtime' + ? propertyOf('VirtualMachine', 'vm-1', 'runtime', RUNTIME) + : propertyOf('VirtualMachine', 'vm-1', 'config', config(vmPathName)) + }, + }, + fetch: async (url, options) => { + const path = url.pathname + requested.push(path) + if (path.endsWith('.vmx')) { + return response({ body: VMX }) + } + if (path.endsWith('.vmsd')) { + return response({ status: vmsdStatus, statusText: 'Not Found', body: '' }) + } + return response({ body: DESCRIPTOR }) + }, + }) + return { esxi, requested } + } + + it('labels a disk with its own channel index', async function () { + const { esxi } = await transferableEsxi() + + const { disks } = await esxi.getTransferableVmMetadata('vm-1') + + assert.equal(disks.length, 1) + // the channel holds no scsi0:0, the disk used to be labelled as such + assert.equal(disks[0].node, 'scsi0:1') + assert.equal(disks[0].datastore, 'ds main') + assert.equal(disks[0].diskPath, 'a.vm/vm-000001.vmdk') + }) + + it('unwraps every value of the config', async function () { + const { esxi } = await transferableEsxi() + + const metadata = await esxi.getTransferableVmMetadata('vm-1') + + assert.equal(metadata.name_label, 'a vm') + assert.equal(metadata.guestId, 'ubuntu64Guest') + // used to be returned as a one element array + assert.equal(metadata.guestFullName, 'Ubuntu Linux (64-bit)') + assert.equal(metadata.firmware, 'uefi') + assert.equal(metadata.powerState, 'poweredOn') + assert.equal(metadata.memory, 2048 * 1024 * 1024) + assert.equal(metadata.nCpus, 2) + assert.equal(metadata.cdrom, true) + assert.deepEqual(metadata.networks, [{ label: 'VM Network', macAddress: '00:0c:29:00:00:01', isGenerated: true }]) + }) + + it('treats a missing vmsd as a VM without snapshot', async function () { + const { esxi } = await transferableEsxi() + + assert.equal((await esxi.getTransferableVmMetadata('vm-1')).snapshots, undefined) + }) + + it('reports a vmx path it cannot parse', async function () { + // destructuring the null match used to throw a TypeError naming nothing + const { esxi } = await transferableEsxi({ vmPathName: 'a.vm/a.vm.vmx' }) + + await assert.rejects(esxi.getTransferableVmMetadata('vm-1'), { + code: 'BAD_VMX_PATH', + message: "can't parse the path of the vmx of the VM vm-1: a.vm/a.vm.vmx", + vmId: 'vm-1', + }) + }) + + it('does not report an unreadable vmsd as a VM without snapshot', async function () { + // silently transferring a full disk instead of a delta is worse than failing + const { esxi } = await transferableEsxi({ vmsdStatus: 403 }) + + await assert.rejects(esxi.getTransferableVmMetadata('vm-1'), { message: /^403 / }) + }) +}) + +describe('keepAlive', function () { + it('touches the session with the cheapest call there is', async function () { + const { esxi, vimClient } = await connectedEsxi({ + responses: { CurrentTime: () => ({ returnval: new Date(0) }) }, + }) + + await esxi.keepAlive() + + // reading the whole `config` of a VM used to be the keep-alive + assert.deepEqual(vimClient.methods, ['CurrentTime']) + assert.deepEqual(vimClient.callsTo('CurrentTime')[0].args, { _this: 'ServiceInstance' }) + }) +}) + +describe('getDataMap', function () { + const BLOCK_LENGTH = 4096 * 512 + + // the thumbprint is the first step of a spawn: failing it sends the code down the fallback + // without binding a port nor writing a temporary file + const failingThumbprint = async () => { + throw new Error('no vddk on this machine') + } + + const DELTA_DESCRIPTOR = `# Disk DescriptorFile +version=1 +CID=d7980f7a +parentCID=aaaaaaaa +createType="vmfsSparse" +parentFileNameHint="vm.vmdk" + +RW 16384 VMFSSPARSE "vm-000001-delta.vmdk" +` + + const cowdExtent = ({ allocated = [0, 2], numGdEntries = 4 } = {}) => { + const header = Buffer.alloc(2048) + header.write('COWD', 0, 'ascii') + header.writeUInt32LE(1, 4) // version + header.writeUInt32LE(3, 8) // flags + header.writeUInt32LE((numGdEntries * BLOCK_LENGTH) / 512, 12) // capacity in sectors + header.writeUInt32LE(1, 16) // one sector per grain + header.writeUInt32LE(4, 20) // the grain directory follows the header + header.writeUInt32LE(numGdEntries, 24) + + const grainDirectory = Buffer.alloc(numGdEntries * 4) + for (const index of allocated) { + // the sector of the grain table of the block, whatever it is + grainDirectory.writeUInt32LE(8 + index, index * 4) + } + return { grainDirectory, header } + } + + it('falls back to reading the metadata of a COWD delta', async function () { + const { grainDirectory, header } = cowdExtent() + const ranges = [] + + const { esxi } = await connectedEsxi({ + fetch: async (url, options) => { + ranges.push([url.pathname, options.headers.Range]) + if (url.pathname.endsWith('-delta.vmdk')) { + // the extent: its header, then its grain directory + return response({ status: 206, body: options.headers.Range === 'bytes=0-2047' ? header : grainDirectory }) + } + return response({ status: 206, body: DELTA_DESCRIPTOR }) + }, + }) + esxi.getServerThumbprint = failingThumbprint + + const dataMap = await esxi.getDataMap('vm-1', 'ds main', 'a.vm/vm-000001.vmdk') + + assert.deepEqual(dataMap, [ + { offset: 0, length: BLOCK_LENGTH, type: 0 }, + { offset: 2 * BLOCK_LENGTH, length: BLOCK_LENGTH, type: 0 }, + ]) + + // an HTTP range is inclusive: these used to ask for one byte too many, three times + assert.deepEqual(ranges, [ + ['/folder/a.vm/vm-000001.vmdk', 'bytes=0-4095'], + ['/folder/a.vm/vm-000001-delta.vmdk', 'bytes=0-2047'], + ['/folder/a.vm/vm-000001-delta.vmdk', 'bytes=2048-2063'], + ]) + }) + + it('reports a delta which is not COWD with a code instead of an assertion', async function () { + const { header } = cowdExtent() + header.write('SESp', 0, 'ascii') + + const { esxi } = await connectedEsxi({ + fetch: async url => + response({ status: 206, body: url.pathname.endsWith('-delta.vmdk') ? header : DELTA_DESCRIPTOR }), + }) + esxi.getServerThumbprint = failingThumbprint + + await assert.rejects(esxi.getDataMap('vm-1', 'ds main', 'a.vm/vm-000001.vmdk'), { code: 'NO_DATA_MAP' }) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs b/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs deleted file mode 100644 index e6b584157c..0000000000 --- a/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import VHDEsxiSeSparse from './VhdEsxiSeSparse.mjs' -import VhdEsxiCowd from './VhdEsxiCowd.mjs' - -export default async function openDeltaVmdkasVhd(datastoreName, path, parentVhd, opts) { - let disposableVhd - if (path.endsWith('-sesparse.vmdk')) { - disposableVhd = await VHDEsxiSeSparse.open(datastoreName, path, parentVhd, opts) - } else if (path.endsWith('-delta.vmdk')) { - disposableVhd = await VhdEsxiCowd.open(datastoreName, path, parentVhd, opts) - } else { - throw new Error(`Vmdk ${path} does not seems to be a delta vmdk.`) - } - await disposableVhd.value.readBlockAllocationTable() - return disposableVhd -} diff --git a/@xen-orchestra/vmware-explorer/package.json b/@xen-orchestra/vmware-explorer/package.json index dbfceae759..5730bf8fa9 100644 --- a/@xen-orchestra/vmware-explorer/package.json +++ b/@xen-orchestra/vmware-explorer/package.json @@ -3,18 +3,15 @@ "private": false, "version": "0.15.0", "name": "@xen-orchestra/vmware-explorer", + "description": "Reads the VMs and the disks of an ESXi host or a vCenter", "dependencies": { + "@vates/async-each": "^1.0.3", "@vates/nbd-client": "^3.5.1", "@vates/node-vsphere-soap": "^2.1.3", - "@vates/read-chunk": "^1.2.1", - "@vates/task": "^0.7.0", "@xen-orchestra/log": "^0.7.2", - "lodash": "^4.18.0", + "promise-toolbox": "^0.21.0", "semver": "^7.7.2", - "undici": "^7.8.0", - "vhd-lib": "^4.16.1", - "xml2js": "^0.6.2", - "xo-vmdk-to-vhd": "^2.5.10" + "undici": "^7.8.0" }, "engines": { "node": ">=20.18.1" @@ -32,6 +29,7 @@ }, "scripts": { "postversion": "npm publish --access public", - "test": "node --test" + "test": "node --test *.test.mjs parsers/*.test.mjs soap/*.test.mjs", + "test-integration": "node --test *.integ.mjs" } } diff --git a/@xen-orchestra/vmware-explorer/parsers/cowd.mjs b/@xen-orchestra/vmware-explorer/parsers/cowd.mjs new file mode 100644 index 0000000000..bd0c2ac87f --- /dev/null +++ b/@xen-orchestra/vmware-explorer/parsers/cowd.mjs @@ -0,0 +1,83 @@ +// https://github.com/libyal/libvmdk/blob/main/documentation/VMWare%20Virtual%20Disk%20Format%20(VMDK).asciidoc#5-the-cowd-sparse-extent-data-file +export const COWD_SECTOR_SIZE = 512 +export const COWD_HEADER_LENGTH = 2048 +// a grain directory entry addresses a grain table, which addresses at most 4096 grains +export const COWD_GRAIN_TABLE_ENTRIES = 4096 +// the `type` of a range of the data map, as the NBD block status of an allocated range +export const BLOCK_TYPE_DATA = 0 + +const unsupported = message => { + const error = new Error(message) + // the caller has no data map to work with, and has to transfer the disk in full + error.code = 'NO_DATA_MAP' + return error +} + +/** + * Header of a COWD sparse extent, the delta format of ESXi 5 and older. + * + * @param {Buffer} buffer - the first {@link COWD_HEADER_LENGTH} bytes of the extent + * @returns {{ blockLength: number, capacity: number, grainDirectoryOffset: number, grainSize: number, numGdEntries: number }} + */ +export function parseCowdHeader(buffer) { + // every field read below fits in the first 28 bytes + if (buffer.length < 28) { + throw unsupported(`a COWD header is at least 28 bytes, got ${buffer.length}`) + } + + const magic = buffer.subarray(0, 4).toString('ascii') + if (magic !== 'COWD') { + // a recent host writes SeSparse deltas: this used to be an assertion error, in the fallback + // path of a failure, which made the real problem unreadable + throw unsupported(`not a COWD extent (magic: ${JSON.stringify(magic)})`) + } + + const version = buffer.readUInt32LE(4) + const capacity = buffer.readUInt32LE(12) * COWD_SECTOR_SIZE + const grainSize = buffer.readUInt32LE(16) + const grainDirectoryOffset = buffer.readUInt32LE(20) * COWD_SECTOR_SIZE + const numGdEntries = buffer.readUInt32LE(24) + + if (version !== 1 || grainSize !== 1) { + // the layout of the grain tables below assumes a grain of one sector + throw unsupported(`unsupported COWD extent, version ${version} with grains of ${grainSize} sector(s)`) + } + + const blockLength = COWD_GRAIN_TABLE_ENTRIES * grainSize * COWD_SECTOR_SIZE + // the value is read from the file: without a bound, a corrupt header asks for a range of several + // gigabytes and builds a map of millions of entries + const maxGdEntries = Math.ceil(capacity / blockLength) + 1 + if (numGdEntries > maxGdEntries) { + throw unsupported( + `COWD extent announces ${numGdEntries} grain directory entries for ${capacity} bytes, at most ${maxGdEntries} are possible` + ) + } + + return { blockLength, capacity, grainDirectoryOffset, grainSize, numGdEntries } +} + +/** + * Blocks of a COWD extent which hold data. + * + * A grain table is only allocated when its block has been written to, so a non zero entry in the + * grain directory is what marks a block as allocated. + * + * @param {Buffer} grainDirectory + * @param {object} params + * @param {number} params.blockLength + * @param {number} params.numGdEntries + * @returns {Array<{ length: number, offset: number, type: number }>} + */ +export function grainDirectoryToDataMap(grainDirectory, { blockLength, numGdEntries }) { + if (grainDirectory.length < numGdEntries * 4) { + throw unsupported(`a grain directory of ${numGdEntries} entries is ${numGdEntries * 4} bytes`) + } + + const dataMap = [] + for (let i = 0; i < numGdEntries; i++) { + if (grainDirectory.readUInt32LE(i * 4) !== 0) { + dataMap.push({ offset: i * blockLength, length: blockLength, type: BLOCK_TYPE_DATA }) + } + } + return dataMap +} diff --git a/@xen-orchestra/vmware-explorer/parsers/cowd.test.mjs b/@xen-orchestra/vmware-explorer/parsers/cowd.test.mjs new file mode 100644 index 0000000000..8ee2f0856e --- /dev/null +++ b/@xen-orchestra/vmware-explorer/parsers/cowd.test.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { + COWD_GRAIN_TABLE_ENTRIES, + COWD_HEADER_LENGTH, + COWD_SECTOR_SIZE, + grainDirectoryToDataMap, + parseCowdHeader, +} from './cowd.mjs' + +const BLOCK_LENGTH = COWD_GRAIN_TABLE_ENTRIES * COWD_SECTOR_SIZE // 2 MiB + +const cowdHeader = ({ + magic = 'COWD', + version = 1, + capacity = 8 * BLOCK_LENGTH, + grainSize = 1, + numGdEntries = 8, +} = {}) => { + const buffer = Buffer.alloc(COWD_HEADER_LENGTH) + buffer.write(magic, 0, 'ascii') + buffer.writeUInt32LE(version, 4) + buffer.writeUInt32LE(3, 8) // flags + buffer.writeUInt32LE(capacity / COWD_SECTOR_SIZE, 12) + buffer.writeUInt32LE(grainSize, 16) + buffer.writeUInt32LE(4, 20) // the grain directory follows the header + buffer.writeUInt32LE(numGdEntries, 24) + return buffer +} + +describe('parseCowdHeader', function () { + it('reads the geometry of the extent', function () { + assert.deepEqual(parseCowdHeader(cowdHeader()), { + blockLength: BLOCK_LENGTH, + capacity: 8 * BLOCK_LENGTH, + grainDirectoryOffset: COWD_HEADER_LENGTH, + grainSize: 1, + numGdEntries: 8, + }) + }) + + it('rejects an extent which is not COWD with a code, not an assertion', function () { + // this is what a recent host writes, and this parser is the fallback of a failure: an + // assertion error there hides the real problem + assert.throws(() => parseCowdHeader(cowdHeader({ magic: 'SESp' })), { + code: 'NO_DATA_MAP', + message: 'not a COWD extent (magic: "SESp")', + }) + }) + + it('rejects a layout it cannot read', function () { + assert.throws(() => parseCowdHeader(cowdHeader({ version: 2 })), { code: 'NO_DATA_MAP' }) + assert.throws(() => parseCowdHeader(cowdHeader({ grainSize: 8 })), { code: 'NO_DATA_MAP' }) + assert.throws(() => parseCowdHeader(Buffer.alloc(8)), { code: 'NO_DATA_MAP' }) + }) + + it('bounds the number of grain directory entries', function () { + // without a bound, this asks the host for a 16 GB range and builds a map of a billion entries + assert.throws(() => parseCowdHeader(cowdHeader({ capacity: BLOCK_LENGTH, numGdEntries: 0xffffffff })), { + code: 'NO_DATA_MAP', + message: /at most 2 are possible$/, + }) + }) +}) + +describe('grainDirectoryToDataMap', function () { + const geometry = { blockLength: BLOCK_LENGTH, numGdEntries: 4 } + + it('keeps the blocks whose grain table is allocated', function () { + const grainDirectory = Buffer.alloc(4 * 4) + grainDirectory.writeUInt32LE(4, 0) // sector of the grain table of the first block + grainDirectory.writeUInt32LE(12, 2 * 4) + + assert.deepEqual(grainDirectoryToDataMap(grainDirectory, geometry), [ + { offset: 0, length: BLOCK_LENGTH, type: 0 }, + { offset: 2 * BLOCK_LENGTH, length: BLOCK_LENGTH, type: 0 }, + ]) + }) + + it('returns an empty map for an extent without any data', function () { + assert.deepEqual(grainDirectoryToDataMap(Buffer.alloc(4 * 4), geometry), []) + }) + + it('refuses a truncated grain directory', function () { + assert.throws(() => grainDirectoryToDataMap(Buffer.alloc(4), geometry), { code: 'NO_DATA_MAP' }) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/soap/VimClient.mjs b/@xen-orchestra/vmware-explorer/soap/VimClient.mjs new file mode 100644 index 0000000000..01690e0151 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/VimClient.mjs @@ -0,0 +1,152 @@ +import { Client } from '@vates/node-vsphere-soap' +import { createLogger } from '@xen-orchestra/log' +import { pTimeout } from 'promise-toolbox' + +const { warn } = createLogger('xo:vmware-explorer:soap') + +const noop = () => {} + +// a vim25 call either answers or fails fast: the long running work is always delegated to a Task, +// whose completion is polled separately +const DEFAULT_CALL_TIMEOUT = 60e3 + +/** + * Promise interface over `@vates/node-vsphere-soap`. + * + * The library exposes one event emitter per call, and another one for the connection itself. Mixing + * both is what made the previous implementation leak a listener per call and reject a call with the + * error of another one, so this class keeps them strictly separate: + * - the connection is a memoized promise + * - every call listens on its own emitter, and removes its listeners in every outcome + */ +export class VimClient { + #client + #connected + + /** + * @param {string} host + * @param {string} user + * @param {string} password + * @param {boolean} sslVerify + * @param {object} [options] + * @param {(error: Error) => void} [options.onError] - called for the errors raised by the + * underlying client once connected, e.g. a session that could not be renewed + * @param {object} [options.client] - injectable `@vates/node-vsphere-soap` client, for tests + */ + constructor(host, user, password, sslVerify, { onError, client } = {}) { + this.#client = client ?? new Client(host, user, password, sslVerify) + + this.#client.on('error', error => { + if (onError !== undefined) { + onError(error) + } else { + warn('unhandled SOAP client error', { error }) + } + }) + } + + get serviceContent() { + return this.#client.serviceContent + } + + get status() { + return this.#client.status + } + + /** + * Resolves once logged in. Memoized: it is safe (and expected) to await it before every call. + * + * @returns {Promise} + */ + connect() { + this.#connected ??= new Promise((resolve, reject) => { + const client = this.#client + + // the client starts connecting in its constructor, the event may already be gone + if (client.status === 'ready') { + return resolve() + } + + const onReady = () => { + client.off('error', onError) + resolve() + } + const onError = error => { + client.off('ready', onReady) + reject(error) + } + + client.once('ready', onReady) + client.once('error', onError) + + if (client.status === 'disconnected') { + // the client connects on its own in its constructor only + client.emit('connect') + } + }).catch(error => { + // a failed connection must not be memoized, or every later call reports an error which may + // not be true any more + this.#connected = undefined + throw error + }) + + return this.#connected + } + + /** + * Runs a vim25 method. + * + * @param {string} method - name of the method, as exposed by the WSDL + * @param {object} [args] - arguments of the method, `_this` included + * @param {object} [options] + * @param {number} [options.timeout] - in ms + * @returns {Promise} the `result` of the call, e.g. `{ returnval }` + */ + async call(method, args, { timeout = DEFAULT_CALL_TIMEOUT } = {}) { + await this.connect() + + const promise = new Promise((resolve, reject) => { + const emitter = this.#client.runCommand(method, args) + + const onResult = result => { + emitter.off('error', onError) + resolve(result) + } + const onError = error => { + emitter.off('result', onResult) + reject(error) + } + + emitter.once('result', onResult) + emitter.once('error', onError) + }) + + return pTimeout.call(promise, timeout) + } + + /** + * Closes the session on the server. + * + * A session is not released when the process ends, it lingers until it expires, and a host only + * accepts a limited number of them. + * + * @returns {Promise} + */ + async close() { + // a close racing the login used to return early and let the login complete afterwards, leaving + // the session open until it expires. The outcome of the connection is not the business of this + // method: it is already reported to whoever awaited `connect()` + await this.#connected?.catch(noop) + + if (this.#client.status !== 'ready') { + return + } + try { + // `runCommand` handles the bookkeeping of a `Logout`: status and exit hook + await this.call('Logout', { _this: this.#client.sessionManager }) + } catch (error) { + // the session will expire on its own, there is nothing better to do + warn('failed to log out', { error }) + } + } +} diff --git a/@xen-orchestra/vmware-explorer/soap/VimClient.test.mjs b/@xen-orchestra/vmware-explorer/soap/VimClient.test.mjs new file mode 100644 index 0000000000..3adbbf813f --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/VimClient.test.mjs @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { EventEmitter, once } from 'node:events' +import { setImmediate as flush } from 'node:timers/promises' + +import { VimClient } from './VimClient.mjs' + +class FakeSoapClient extends EventEmitter { + constructor({ status = 'ready' } = {}) { + super() + this.status = status + this.sessionManager = 'SessionManager' + this.serviceContent = { propertyCollector: 'propertyCollector' } + this.calls = [] + } + + runCommand(command, args) { + const emitter = new EventEmitter() + this.calls.push({ command, args, emitter }) + return emitter + } + + get lastCall() { + return this.calls[this.calls.length - 1] + } +} + +const makeClient = (options = {}) => { + const client = new FakeSoapClient(options) + return { client, vimClient: new VimClient('host', 'user', 'password', true, { ...options, client }) } +} + +describe('VimClient', function () { + describe('connect', function () { + it('resolves immediately when the client is already connected', async function () { + const { vimClient } = makeClient() + await vimClient.connect() + }) + + it('waits for the ready event', async function () { + const { client, vimClient } = makeClient({ status: 'connecting' }) + const promise = vimClient.connect() + client.emit('ready') + await promise + }) + + it('rejects with the connection error', async function () { + const { client, vimClient } = makeClient({ status: 'connecting' }) + const error = new Error('login failed') + const promise = vimClient.connect() + client.emit('error', error) + await assert.rejects(promise, /login failed/) + }) + + it('does not memoize a failed connection', async function () { + const { client, vimClient } = makeClient({ status: 'connecting' }) + const first = vimClient.connect() + client.emit('error', new Error('login failed')) + await assert.rejects(first, /login failed/) + + // the client is disconnected, connecting again must trigger a new attempt + client.status = 'disconnected' + const connecting = once(client, 'connect') + const second = vimClient.connect() + await connecting + client.status = 'ready' + client.emit('ready') + + await second + assert.notEqual(first, second) + }) + + it('is memoized', async function () { + const { client, vimClient } = makeClient() + assert.equal(vimClient.connect(), vimClient.connect()) + // no listener is left on the client, besides the one forwarding errors + assert.equal(client.listenerCount('ready'), 0) + }) + }) + + describe('call', function () { + it('resolves with the result of the call', async function () { + const { client, vimClient } = makeClient() + const promise = vimClient.call('PowerOffVM_Task', { _this: 'vm-1' }) + + await flush() // let the call go through the awaited connection + const { command, args, emitter } = client.lastCall + assert.equal(command, 'PowerOffVM_Task') + assert.deepEqual(args, { _this: 'vm-1' }) + emitter.emit('result', { returnval: { $value: 'task-1' } }) + + assert.deepEqual(await promise, { returnval: { $value: 'task-1' } }) + }) + + it('rejects with the error of its own call', async function () { + const { client, vimClient } = makeClient() + const promise = vimClient.call('Whatever') + + await flush() + client.lastCall.emitter.emit('error', new Error('InvalidDeviceSpec')) + + await assert.rejects(promise, /InvalidDeviceSpec/) + }) + + it('is not disturbed by the failure of a concurrent call', async function () { + const { client, vimClient } = makeClient() + const failing = vimClient.call('Failing') + const succeeding = vimClient.call('Succeeding') + + await flush() + const [first, second] = client.calls + first.emitter.emit('error', new Error('nope')) + second.emitter.emit('result', 'ok') + + await assert.rejects(failing, /nope/) + assert.equal(await succeeding, 'ok') + }) + + it('does not leak listeners', async function () { + const { client, vimClient } = makeClient() + + for (let i = 0; i < 20; i++) { + const promise = vimClient.call(`Call${i}`) + await flush() + const { emitter } = client.lastCall + if (i % 2 === 0) { + emitter.emit('result', i) + assert.equal(await promise, i) + } else { + emitter.emit('error', new Error(`error ${i}`)) + await assert.rejects(promise) + } + assert.equal(emitter.listenerCount('result'), 0) + assert.equal(emitter.listenerCount('error'), 0) + } + + // the only listener on the client is the one forwarding its errors + assert.equal(client.listenerCount('error'), 1) + assert.equal(client.listenerCount('result'), 0) + }) + + it('rejects when the server does not answer', async function () { + const { vimClient } = makeClient() + await assert.rejects(vimClient.call('Silent', undefined, { timeout: 1 }), { name: 'TimeoutError' }) + }) + }) + + describe('close', function () { + it('logs out', async function () { + const { client, vimClient } = makeClient() + const promise = vimClient.close() + + await flush() + assert.equal(client.lastCall.command, 'Logout') + assert.deepEqual(client.lastCall.args, { _this: 'SessionManager' }) + client.lastCall.emitter.emit('result', {}) + + await promise + }) + + it('does nothing when not connected', async function () { + const { client, vimClient } = makeClient({ status: 'disconnected' }) + await vimClient.close() + assert.equal(client.calls.length, 0) + }) + + it('waits for a login in flight before logging out', async function () { + const { client, vimClient } = makeClient({ status: 'connecting' }) + const connecting = vimClient.connect() + + // this used to return early on a status which was not 'ready' yet: the login then completed + // afterwards and its session lingered on the server until it expired + const closing = vimClient.close() + await flush() + assert.deepEqual(client.calls, []) + + client.status = 'ready' + client.emit('ready') + await connecting + + await flush() + assert.equal(client.lastCall.command, 'Logout') + client.lastCall.emitter.emit('result', {}) + + await closing + }) + + it('does not log out when the login it waited for failed', async function () { + const { client, vimClient } = makeClient({ status: 'connecting' }) + const connecting = vimClient.connect() + const closing = vimClient.close() + + client.emit('error', new Error('login failed')) + await assert.rejects(connecting, /login failed/) + + // the failure belongs to the caller of `connect`, closing has nothing to report + await closing + assert.deepEqual(client.calls, []) + }) + + it('does not throw when the logout fails', async function () { + const { client, vimClient } = makeClient() + const promise = vimClient.close() + + await flush() + client.lastCall.emitter.emit('error', new Error('session already gone')) + + await promise + }) + }) + + it('forwards the errors raised once connected', async function () { + const client = new FakeSoapClient() + const errors = [] + // eslint-disable-next-line no-new + new VimClient('host', 'user', 'password', true, { client, onError: error => errors.push(error) }) + + const error = new Error('session expired') + client.emit('error', error) + + assert.deepEqual(errors, [error]) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/soap/normalize.mjs b/@xen-orchestra/vmware-explorer/soap/normalize.mjs new file mode 100644 index 0000000000..ab3cab0c74 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/normalize.mjs @@ -0,0 +1,40 @@ +/** + * The SOAP library returns a single element as the element itself, and a repeated one as an array. + * + * @param {unknown} value + * @returns {Array} - an empty array when the element is absent + */ +export function asArray(value) { + if (value === undefined || value === null) { + return [] + } + return Array.isArray(value) ? value : [value] +} + +/** + * Unwraps a value returned by the SOAP library. + * + * A value carrying attributes is wrapped: a scalar becomes `{ $value, attributes }` and a complex + * type gets an extra `attributes` property. Both are noise for the callers, which only care about + * the value itself. + * + * The normalization is intentionally shallow: the library only wraps the elements which do carry + * attributes, so nested scalars are already plain values. + * + * @param {unknown} value + * @returns {unknown} + */ +export function normalizeSoapValue(value) { + if (value === null || typeof value !== 'object') { + return value + } + // a falsy scalar ( '', 0, false ) is still a value + if ('$value' in value) { + return value.$value + } + if ('attributes' in value) { + const { attributes, ...rest } = value + return rest + } + return value +} diff --git a/@xen-orchestra/vmware-explorer/soap/normalize.test.mjs b/@xen-orchestra/vmware-explorer/soap/normalize.test.mjs new file mode 100644 index 0000000000..27170dc942 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/normalize.test.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { asArray, normalizeSoapValue } from './normalize.mjs' + +describe('asArray', function () { + it('wraps a single element', function () { + assert.deepEqual(asArray('a'), ['a']) + assert.deepEqual(asArray({ a: 1 }), [{ a: 1 }]) + }) + + it('keeps an array as is', function () { + const array = ['a', 'b'] + assert.equal(asArray(array), array) + }) + + it('returns an empty array for an absent element', function () { + assert.deepEqual(asArray(undefined), []) + assert.deepEqual(asArray(null), []) + }) +}) + +describe('normalizeSoapValue', function () { + it('unwraps a scalar', function () { + assert.equal(normalizeSoapValue({ $value: 'poweredOn', attributes: { 'xsi:type': 'string' } }), 'poweredOn') + }) + + it('unwraps a falsy scalar', function () { + assert.equal(normalizeSoapValue({ $value: '' }), '') + assert.equal(normalizeSoapValue({ $value: 0 }), 0) + assert.equal(normalizeSoapValue({ $value: false }), false) + }) + + it('unwraps a managed object reference to its value', function () { + assert.equal(normalizeSoapValue({ attributes: { type: 'Datastore' }, $value: 'datastore-11' }), 'datastore-11') + }) + + it('drops the attributes of a complex type', function () { + assert.deepEqual( + normalizeSoapValue({ attributes: { 'xsi:type': 'VirtualMachineConfigInfo' }, name: 'vm', guestId: 'ubuntu64' }), + { name: 'vm', guestId: 'ubuntu64' } + ) + }) + + it('does not touch a value without attributes', function () { + const value = { name: 'vm' } + assert.equal(normalizeSoapValue(value), value) + }) + + it('leaves the nested values as they are', function () { + assert.deepEqual(normalizeSoapValue({ attributes: {}, hardware: { memoryMB: '2048' } }), { + hardware: { memoryMB: '2048' }, + }) + }) + + it('passes primitives and null through', function () { + assert.equal(normalizeSoapValue('poweredOff'), 'poweredOff') + assert.equal(normalizeSoapValue(null), null) + assert.equal(normalizeSoapValue(undefined), undefined) + }) +}) diff --git a/@xen-orchestra/vmware-explorer/soap/specs.mjs b/@xen-orchestra/vmware-explorer/soap/specs.mjs new file mode 100644 index 0000000000..8f42d1e4c4 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/specs.mjs @@ -0,0 +1,81 @@ +/** + * Builders for the vim25 arguments we hand to the SOAP library. + * + * The schema declares every complex type as an `xs:sequence`, so the server rejects a spec whose + * children are out of order — and the library derives that order from JS key insertion order, + * which is exactly the kind of thing that breaks silently when an object is refactored or when a + * field is renamed. Declaring the order explicitly here makes it reviewable and testable. + * + * Each list is the full schema order; only the elements which are actually set get emitted. + */ + +export const PROPERTY_SPEC = Object.freeze(['type', 'all', 'pathSet']) + +export const TRAVERSAL_SPEC = Object.freeze(['name', 'type', 'path', 'skip', 'selectSet']) + +export const OBJECT_SPEC = Object.freeze(['obj', 'skip', 'selectSet']) + +export const PROPERTY_FILTER_SPEC = Object.freeze(['propSet', 'objectSet', 'reportMissingObjectsInResults']) + +export const RETRIEVE_OPTIONS = Object.freeze(['maxObjects']) + +/** + * A managed object reference. + * + * @param {string} type - type of the object, e.g. `VirtualMachine` + * @param {string} value - the reference itself, e.g. `vm-42` + * @returns {object} + */ +export function moRef(type, value) { + return { attributes: { type }, $value: value } +} + +/** + * Builds an element, its children in schema order, skipping the values which are not set. + * + * @param {ReadonlyArray} order - schema element order + * @param {string | undefined} xsiType - type to declare, mandatory wherever the schema declares an + * abstract type, e.g. the `selectSet` of an ObjectSpec + * @param {object} values + * @returns {object} + */ +export function orderedChildren(order, xsiType, values) { + const unknown = Object.keys(values).filter(name => values[name] !== undefined && !order.includes(name)) + if (unknown.length > 0) { + // a typo in a field name would otherwise be silently dropped from the request + const error = new Error(`unknown element(s) for this type: ${unknown.join(', ')}`) + // this spec is built here, not by the host: such a failure is a bug of this package + error.code = 'BAD_VIM25_SPEC' + throw error + } + + // the attributes are not an element, but the library expects them first + const node = xsiType === undefined ? {} : { attributes: { 'xsi:type': xsiType } } + for (const name of order) { + const value = values[name] + if (value !== undefined && value !== null) { + node[name] = value + } + } + return node +} + +export function propertySpec(values) { + return orderedChildren(PROPERTY_SPEC, 'PropertySpec', values) +} + +export function traversalSpec(values) { + return orderedChildren(TRAVERSAL_SPEC, 'TraversalSpec', values) +} + +export function objectSpec(values) { + return orderedChildren(OBJECT_SPEC, 'ObjectSpec', values) +} + +export function propertyFilterSpec(values) { + return orderedChildren(PROPERTY_FILTER_SPEC, 'PropertyFilterSpec', values) +} + +export function retrieveOptions(values) { + return orderedChildren(RETRIEVE_OPTIONS, 'RetrieveOptions', values) +} diff --git a/@xen-orchestra/vmware-explorer/soap/specs.test.mjs b/@xen-orchestra/vmware-explorer/soap/specs.test.mjs new file mode 100644 index 0000000000..b34af0b2e9 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/soap/specs.test.mjs @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { + moRef, + objectSpec, + orderedChildren, + propertyFilterSpec, + propertySpec, + retrieveOptions, + traversalSpec, +} from './specs.mjs' + +describe('moRef', function () { + it('carries the type of the object', function () { + assert.deepEqual(moRef('VirtualMachine', 'vm-42'), { attributes: { type: 'VirtualMachine' }, $value: 'vm-42' }) + }) +}) + +describe('orderedChildren', function () { + const ORDER = Object.freeze(['first', 'second', 'third']) + + it('emits the keys in schema order, whatever the order they are given in', function () { + const node = orderedChildren(ORDER, 'SomeType', { third: 3, first: 1 }) + + // the insertion order of the keys is what the library turns into element order + assert.deepEqual(Object.keys(node), ['attributes', 'first', 'third']) + assert.deepEqual(node, { attributes: { 'xsi:type': 'SomeType' }, first: 1, third: 3 }) + }) + + it('skips the values which are not set', function () { + assert.deepEqual(orderedChildren(ORDER, undefined, { first: undefined, second: null, third: 0 }), { third: 0 }) + }) + + it('refuses an unknown element', function () { + assert.throws(() => orderedChildren(ORDER, 'SomeType', { frist: 1 }), { + code: 'BAD_VIM25_SPEC', + message: 'unknown element(s) for this type: frist', + }) + }) + + it('ignores an unknown element which is not set', function () { + assert.deepEqual(orderedChildren(ORDER, undefined, { frist: undefined }), {}) + }) + + it('omits the type when there is none to declare', function () { + assert.deepEqual(Object.keys(orderedChildren(ORDER, undefined, { first: 1 })), ['first']) + }) +}) + +describe('property collector specs', function () { + it('builds a PropertySpec', function () { + assert.deepEqual(propertySpec({ pathSet: ['config'], type: 'VirtualMachine' }), { + attributes: { 'xsi:type': 'PropertySpec' }, + type: 'VirtualMachine', + pathSet: ['config'], + }) + }) + + it('builds an ObjectSpec', function () { + const spec = objectSpec({ skip: false, obj: moRef('Task', 'task-1') }) + + assert.deepEqual(Object.keys(spec), ['attributes', 'obj', 'skip']) + assert.equal(spec.skip, false) + }) + + it('builds a TraversalSpec', function () { + assert.deepEqual(Object.keys(traversalSpec({ path: 'view', name: 'traverseEntities', type: 'ContainerView' })), [ + 'attributes', + 'name', + 'type', + 'path', + ]) + }) + + it('builds a PropertyFilterSpec', function () { + assert.deepEqual( + Object.keys(propertyFilterSpec({ objectSet: [objectSpec({ obj: moRef('Task', 'task-1') })], propSet: [] })), + ['attributes', 'propSet', 'objectSet'] + ) + }) + + it('builds RetrieveOptions', function () { + assert.deepEqual(retrieveOptions({ maxObjects: 100 }), { + attributes: { 'xsi:type': 'RetrieveOptions' }, + maxObjects: 100, + }) + }) +}) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 975abd6bd3..455e683261 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -40,6 +40,15 @@ - [REST API/SDN Controller] Fix deleting a non-existent traffic rule wrongly returning success instead of a 404 (PR [#9895](https://github.com/vatesfr/xen-orchestra/pull/9895)) - [xo-server] Fix a memory leak when a client stops reading a proxied response, e.g. a Prometheus scrape of `/openmetrics` reaching its timeout: the request to the proxied service was never closed and its whole response stayed in memory, which could end up in the appliance being OOM-killed (PR [#10388](https://github.com/vatesfr/xen-orchestra/pull/10388)) - [REST API] Rolling pool update and rolling pool reboot are now refused while a backup job runs on the pool, like their JSON-RPC counterparts, unless `bypassBackupCheck` is set in the request body (PR [#10313](https://github.com/vatesfr/xen-orchestra/pull/10313)) +- [V2V] Consolidating or removing the snapshots of a VM no longer fails with `task execution failed` after one minute: an operation is now given the time it needs (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Fix a failure to read the snapshots of a VM silently transferring the whole disk instead of a delta (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Fix a VM whose controller has no disk on its first slot being imported with a duplicated disk, or in full instead of a delta (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Fix a second import, or the final pass of a warm migration, failing on a disk export server which had already been stopped (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Fix xo-server stopping when `nbdkit` is not installed, or when the connection to the source host is lost during a migration (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Listing the VMs of a host no longer fails entirely when one of them is incomplete, e.g. while being created (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] The credentials are no longer sent again on every read of a disk, and a session expiring during a long import is renewed (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] The session opened on the source host is now closed when a listing, a migration or a disk export ends, instead of piling up until it expires — a host only accepts a limited number of them (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) +- [V2V] Report what the source host complained about, instead of `task execution failed` or an assertion error, when an operation fails (PR [#10363](https://github.com/vatesfr/xen-orchestra/pull/10363)) ### Packages to release @@ -57,12 +66,14 @@ +- @vates/node-vsphere-soap minor - @vates/types minor - @xen-orchestra/backup-archive patch - @xen-orchestra/backups patch - @xen-orchestra/disk-cli patch - @xen-orchestra/qcow2 minor - @xen-orchestra/rest-api minor +- @xen-orchestra/vmware-explorer major - @xen-orchestra/web minor - @xen-orchestra/web-core minor - @xen-orchestra/xapi patch diff --git a/packages/xo-server/src/xo-mixins/vmware/importDisksfromDatastore.mjs b/packages/xo-server/src/xo-mixins/vmware/importDisksfromDatastore.mjs index 2f9e30486d..0febe957a9 100644 --- a/packages/xo-server/src/xo-mixins/vmware/importDisksfromDatastore.mjs +++ b/packages/xo-server/src/xo-mixins/vmware/importDisksfromDatastore.mjs @@ -1,3 +1,4 @@ +import { Disposable } from 'promise-toolbox' import { Task } from '@xen-orchestra/mixins/Tasks.mjs' import { QCOW2_CLUSTER_SIZE, VDI_FORMAT_QCOW2, VDI_FORMAT_VHD, VHD_BLOCK_SIZE, VHD_MAX_SIZE } from '@xen-orchestra/xapi' import { ReadAhead } from '@xen-orchestra/disk-transform' @@ -19,37 +20,42 @@ export async function importStream({ esxi, dataMap, disk, vmId, format }, consum const signal = Task.abortSignal const { datastore: datastoreName, diskPath } = disk - let vmdk - let stream try { + // the server is stopped whatever happens here: it used to be killed by a `finally` which had + // to name the disk and repeat the exact spawn options // we read the data from the full chain to ensure we don't have partial blocks ( blocks with 0 when clusters are in parent only) - const { nbdInfos } = await esxi.spawnNbdKitProcess(vmId, `[${datastoreName}] ${diskPath}`) - signal?.throwIfAborted() - vmdk = new NbdDisk(nbdInfos, READ_BLOCK_SIZE, { dataMap }) + return await Disposable.use(esxi.getNbdServer(vmId, `[${datastoreName}] ${diskPath}`), async ({ nbdInfos }) => { + let vmdk + let stream + try { + signal?.throwIfAborted() + vmdk = new NbdDisk(nbdInfos, READ_BLOCK_SIZE, { dataMap }) - await vmdk.init() - signal?.throwIfAborted() - vmdk = new ReadAhead(vmdk) + await vmdk.init() + signal?.throwIfAborted() + vmdk = new ReadAhead(vmdk) - vmdk.addProgressHandler(new TaskProgressHandler()) + vmdk.addProgressHandler(new TaskProgressHandler()) - if (format === VDI_FORMAT_QCOW2) { - stream = await toQcow2Stream(vmdk, { signal }) - } else { - stream = await toVhdStream(vmdk, { signal }) - } - Task.info(`got source stream for ${diskPath}`) - await consumerCallback(stream) - return Math.round((vmdk.getNbGeneratedBlock() * vmdk.getBlockSize()) / 1024 / 1024) + if (format === VDI_FORMAT_QCOW2) { + stream = await toQcow2Stream(vmdk, { signal }) + } else { + stream = await toVhdStream(vmdk, { signal }) + } + Task.info(`got source stream for ${diskPath}`) + await consumerCallback(stream) + return Math.round((vmdk.getNbGeneratedBlock() * vmdk.getBlockSize()) / 1024 / 1024) + } catch (err) { + stream?.destroy(err) + throw err + } finally { + await vmdk?.close().catch(err => warn('error while closing source vmdk', err)) + } + }) } catch (err) { - stream?.destroy(err) + // a failure to start the server belongs in the report too, e.g. nbdkit not being installed Task.warning(err) throw err - } finally { - await vmdk?.close().catch(err => warn('error while closing source vmdk', err)) - await esxi - .killNbdServer(vmId, `[${datastoreName}] ${diskPath}`) - .catch(err => warn('error while stopping nbdkit server', err)) } } diff --git a/packages/xo-server/src/xo-mixins/vmware/index.mjs b/packages/xo-server/src/xo-mixins/vmware/index.mjs index 260e4d912f..ef4de950a9 100644 --- a/packages/xo-server/src/xo-mixins/vmware/index.mjs +++ b/packages/xo-server/src/xo-mixins/vmware/index.mjs @@ -23,14 +23,31 @@ export default class MigrateVm { #connectToEsxi(host, user, password, sslVerify) { return Task.run({ properties: { name: `connecting to ${host}` } }, async () => { const esxi = new Esxi(host, user, password, sslVerify) - await fromEvent(esxi, 'ready') + try { + await fromEvent(esxi, 'ready') + } catch (error) { + // the login may have succeeded even though connecting did not, e.g. when it is listing the + // datacenters which failed: that session would linger on the host until it expires + await this.#closeEsxi(esxi, host) + throw error + } return esxi }) } + // a session is not released when this process ends, it lingers on the host until it expires, and + // a host only accepts a limited number of them. Failing to close is not worth failing a job + #closeEsxi(esxi, host) { + return esxi.close().catch(error => warn('failed to close the ESXi session', { error, host })) + } + async connectToEsxiAndList({ host, user, password, sslVerify }) { const esxi = await this.#connectToEsxi(host, user, password, sslVerify) - return esxi.getAllVmMetadata() + try { + return await esxi.getAllVmMetadata() + } finally { + await this.#closeEsxi(esxi, host) + } } async #findBaseVM(xapi, metadata) { @@ -156,10 +173,9 @@ export default class MigrateVm { }) // ensure the vmware session stays alive const interval = setInterval( - async () => { - try { - await esxi.fetchProperty('VirtualMachine', vmId, 'config') - } catch (_) {} + () => { + // a failure here is not fatal: the client logs in again when a call finds the session gone + esxi.keepAlive().catch(error => warn('failed to keep the ESXi session alive', { error, vmId })) }, 15 * 60 * 1000 ) @@ -200,6 +216,8 @@ export default class MigrateVm { ) { const app = this._app const esxi = await this.#connectToEsxi(host, user, password, sslVerify) + // every transfer is awaited below, nothing is left reading from the host when this returns + $defer(() => this.#closeEsxi(esxi, host)) const sr = app.getXapiObject(srId) const template = app.getXapiObject(templateId) const xapi = sr.$xapi @@ -276,6 +294,7 @@ export default class MigrateVm { } const disk = chain.pop() const stream = new PassThrough() + // the transfer outlives this call, so the session cannot be released before the stream is done importStream({ esxi, disk, vmId, format }, source => { stream.length = source.length source.pipe(stream) @@ -287,10 +306,14 @@ export default class MigrateVm { error ? reject(error) : resolve() }) }) - }).catch(error => { - warn('Error while reading the disk from esxi', warn) - throw error }) + .catch(error => { + // rethrowing was an unhandled rejection: nothing awaits this promise. Whoever reads the + // stream is the one which has to learn about the failure + warn('error while reading the disk from esxi', { diskId, error, vmId }) + stream.destroy(error) + }) + .finally(() => this.#closeEsxi(esxi, host)) return stream }