Compare commits

..

3 Commits

Author SHA1 Message Date
Florent BEAUCHAMP
388125f637 better type handling 2026-11-10 09:49:37 -05:00
Florent BEAUCHAMP
5676dad08d feat(rest-api): expose sort on all endpoints 2026-11-01 08:12:53 -05:00
Florent BEAUCHAMP
4f098e1455 feat(rest-api): implement sort 2026-11-01 08:11:26 -05:00
1420 changed files with 8073 additions and 63892 deletions

View File

@@ -53,12 +53,6 @@ module.exports = {
],
},
},
{
files: ['@xen-orchestra/web-core/**/*.unit.{test,spec}.ts'],
rules: {
'n/no-unpublished-import': 'off',
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/**/*.{vue,ts}'],
parser: 'vue-eslint-parser',

View File

@@ -44,39 +44,3 @@ obfuscated.token = 'qux'
merge(obfuscated, original)
// → { username: 'foo', password: 'bar', token: 'qux' }
```
### `redact(value, [isSensitive], [options])`
> Recursively replace sensitive entries with the same special tag as `obfuscate()`, whatever their type.
>
> Unlike `obfuscate()`, it uses a broader list of sensitive keys (`isExportSensitiveParam`) because redacted values are never merged back: it is meant for data leaving XO, such as support dumps or audit records. Keys are kept so that the structure stays usable.
Options:
- `replacement`: the special tag of `obfuscate()` by default
- `urls`: whether to also apply `redactUrls()` to the other strings, `false` by default
```js
import { redact } from '@vates/obfuscate'
redact({ username: 'foo', password: 'bar', cluster_token: { value: 'baz' } })
// → {
// username: 'foo',
// password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// cluster_token: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// }
redact({ error: 'cannot mount nfs://user:pass@host/path' }, undefined, { replacement: '*', urls: true })
// → { error: 'cannot mount nfs://*@host/path' }
```
### `redactUrls(string, [replacement])`
> Replace the credentials and the sensitive query parameters (e.g. `encryptionKey`) of the URLs in a string, wherever they are (e.g. in an error message).
```js
import { redactUrls } from '@vates/obfuscate'
redactUrls('s3://KEY:secret@s3.lan/bucket?encryptionKey=foo', '*')
// → 's3://*@s3.lan/bucket?encryptionKey=*'
```

View File

@@ -61,42 +61,6 @@ merge(obfuscated, original)
// → { username: 'foo', password: 'bar', token: 'qux' }
```
### `redact(value, [isSensitive], [options])`
> Recursively replace sensitive entries with the same special tag as `obfuscate()`, whatever their type.
>
> Unlike `obfuscate()`, it uses a broader list of sensitive keys (`isExportSensitiveParam`) because redacted values are never merged back: it is meant for data leaving XO, such as support dumps or audit records. Keys are kept so that the structure stays usable.
Options:
- `replacement`: the special tag of `obfuscate()` by default
- `urls`: whether to also apply `redactUrls()` to the other strings, `false` by default
```js
import { redact } from '@vates/obfuscate'
redact({ username: 'foo', password: 'bar', cluster_token: { value: 'baz' } })
// → {
// username: 'foo',
// password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// cluster_token: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// }
redact({ error: 'cannot mount nfs://user:pass@host/path' }, undefined, { replacement: '*', urls: true })
// → { error: 'cannot mount nfs://*@host/path' }
```
### `redactUrls(string, [replacement])`
> Replace the credentials and the sensitive query parameters (e.g. `encryptionKey`) of the URLs in a string, wherever they are (e.g. in an error message).
```js
import { redactUrls } from '@vates/obfuscate'
redactUrls('s3://KEY:secret@s3.lan/bucket?encryptionKey=foo', '*')
// → 's3://*@s3.lan/bucket?encryptionKey=*'
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on

View File

@@ -57,74 +57,3 @@ function replace(value, replacement) {
return helper(value)
}
exports.replace = replace
// broader than SENSITIVE_PARAMS because redacted values are never merged back:
// meant for data leaving XO (e.g. support dumps, audit records), not for display
const EXPORT_SENSITIVE_PARAMS = [
/api[-_]?key/i, // e.g. `apikey` in the XenServer credentials of user preferences
'encryptionKey',
'privateKey',
'pw_hash',
/passphrase/i,
/password/i,
/secret/i,
/token/i,
]
const isExportSensitiveParam = name =>
EXPORT_SENSITIVE_PARAMS.some(pattern => (typeof pattern === 'string' ? pattern === name : pattern.test(name)))
exports.isExportSensitiveParam = isExportSensitiveParam
// SMB URLs (e.g. of backup repositories) contain the raw password, which can
// contain any character (e.g. `smb://user:p@ss/w@domain\\\\host\0path`), so the
// credentials are everything up to the last `@` before the path
const SMB_CREDENTIALS_RE = /^(smb:\/\/)[^\0]*@/i
// other URLs, anywhere in the string (e.g. an error message): credentials are
// URL encoded and end at the first `@`, see https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.1
const URL_CREDENTIALS_RE = /([a-z][a-z\d+.-]*:\/\/)[^\s/?#@]*@/gi
// e.g. `?encryptionKey=` in a backup repository URL, same names as SENSITIVE_PARAMS
const SENSITIVE_QUERY_PARAM_RE = /([?&](?:encryptionKey|passphrase|token|[^=&#\s]*password[^=&#\s]*)=)[^&#\s]*/gi
// replaces the credentials and the sensitive query parameters of the URLs in a string
function redactUrls(string, replacement = OBFUSCATED_VALUE) {
if (!string.includes('://')) {
return string
}
return string
.replace(SMB_CREDENTIALS_RE, (_, scheme) => `${scheme}${replacement}@`)
.replace(URL_CREDENTIALS_RE, (_, scheme) => `${scheme}${replacement}@`)
.replace(SENSITIVE_QUERY_PARAM_RE, (_, param) => `${param}${replacement}`)
}
exports.redactUrls = redactUrls
// unlike replace(), also replaces non-string entries (e.g. a whole object under
// a `secret` key), keys are kept so that the structure stays usable
//
// options:
// - `replacement`: OBFUSCATED_VALUE by default
// - `urls`: whether to apply redactUrls() to the other strings, false by default
function redact(value, isSensitive = isExportSensitiveParam, { replacement = OBFUSCATED_VALUE, urls = false } = {}) {
function helper(value) {
if (typeof value === 'string') {
return urls ? redactUrls(value, replacement) : value
}
if (typeof value !== 'object' || value === null) {
return value
}
if (Array.isArray(value)) {
return value.map(helper)
}
const result = {}
for (const key of Object.keys(value)) {
result[key] = isSensitive(key) ? replacement : helper(value[key])
}
return result
}
return helper(value)
}
exports.redact = redact

View File

@@ -1,94 +0,0 @@
'use strict'
const { describe, it } = require('node:test')
const assert = require('node:assert/strict')
const { obfuscate, OBFUSCATED_VALUE, redact, redactUrls } = require('./')
describe('redact()', () => {
it('replaces sensitive entries at any depth, whatever their type, and keeps the keys', () => {
const remote = {
name: 'nas',
url: 's3://key@host/bucket',
encryptionKey: 'aes-key',
proxy: { authenticationToken: 'proxy-token' },
credentials: [{ username: 'admin', password: 'pass', secret: { value: 'chap' } }],
}
assert.deepEqual(redact(remote), {
name: 'nas',
url: 's3://key@host/bucket',
encryptionKey: OBFUSCATED_VALUE,
proxy: { authenticationToken: OBFUSCATED_VALUE },
credentials: [{ username: 'admin', password: OBFUSCATED_VALUE, secret: OBFUSCATED_VALUE }],
})
})
it('catches keys missed by obfuscate()', () => {
const plugin = { apiKey: 'k', cluster_token: 't', secretAccessKey: 's', pw_hash: 'h' }
assert.deepEqual(redact(plugin), {
apiKey: OBFUSCATED_VALUE,
cluster_token: OBFUSCATED_VALUE,
secretAccessKey: OBFUSCATED_VALUE,
pw_hash: OBFUSCATED_VALUE,
})
assert.equal(obfuscate(plugin).apiKey, 'k')
})
it('accepts a custom predicate and replacement', () => {
assert.deepEqual(
redact({ a: 1, b: { a: 2, c: 3 } }, key => key === 'a', { replacement: '*' }),
{
a: '*',
b: { a: '*', c: 3 },
}
)
})
it('redacts the URLs of the other strings with `urls`', () => {
const data = {
params: { url: 'smb://admin:p@ss@nas\\\\share' },
error: 'mount of nfs://user:pass@host/path failed',
}
assert.deepEqual(redact(data, undefined, { replacement: '*', urls: true }), {
params: { url: 'smb://*@nas\\\\share' },
error: 'mount of nfs://*@host/path failed',
})
assert.deepEqual(redact(data), data)
})
it('returns other values as is', () => {
assert.equal(redact('password'), 'password')
assert.equal(redact(null), null)
})
})
describe('redactUrls()', () => {
it('replaces the credentials of URLs, anywhere in the string', () => {
assert.equal(
redactUrls('failed to connect to https://user:p%40ss@host/ and s3://KEY:secret@s3.lan/bucket'),
`failed to connect to https://${OBFUSCATED_VALUE}@host/ and s3://${OBFUSCATED_VALUE}@s3.lan/bucket`
)
})
it('replaces the credentials of SMB URLs up to the last @ before the path', () => {
assert.equal(redactUrls('smb://user:p@ss/w@domain\\\\host\0path', '*'), 'smb://*@domain\\\\host\0path')
})
it('replaces sensitive query parameters', () => {
assert.equal(
redactUrls('s3://host/bucket?encryptionKey=k&region=eu&adminPassword=p', '*'),
's3://host/bucket?encryptionKey=*&region=eu&adminPassword=*'
)
})
it('handles replacements containing $', () => {
assert.equal(redactUrls('http://u:p@h/', '$1'), 'http://$1@h/')
})
it('returns strings without URLs as is', () => {
assert.equal(redactUrls('user:pass@host'), 'user:pass@host')
})
})

View File

@@ -18,7 +18,6 @@
"node": ">=8.10"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node --test"
"postversion": "npm publish --access public"
}
}

View File

@@ -1,48 +0,0 @@
```js
import { createClient } from '@vates/syslog'
const client = createClient('tls://syslog.company.lan', {
appName: 'my-app',
facility: 'authpriv',
})
await client.send('something happened', { msgid: 'event', severity: 'warning' })
// closes the connection once the messages already sent are delivered, the client reconnects if it is used again
client.close()
```
### `createClient(uri, options)`
`uri`: `udp://host[:port]`, `tcp://host[:port]` or `tls://host[:port]`, the default port is `514`, or `6514` for TLS.
Options:
- `appName`: application name of the messages, `node` by default
- `facility`: one of `FACILITIES` (e.g. `user`, `authpriv`, `local0`), `user` by default
- `format`: `rfc5424` (default) or `rfc3164`
- `framing` (TCP and TLS): `lf` (default, each message followed by a line feed, its own line breaks written as `\n` and `\r`) or `octet-counting` (each message prefixed by its length, required by some servers for TLS, see [RFC 5425](https://datatracker.ietf.org/doc/html/rfc5425#section-4.3))
- `hostname`: host name of the messages, `os.hostname()` by default
- `ca` (TLS): contents of PEM certificate authorities to trust, in addition to the system ones
- `allowUnauthorized` (TLS): do not check the certificate of the server, `false` by default
- `closeTimeout`: in milliseconds, how long `close()` waits for the messages already sent, `10e3` by default
- `connectTimeout`: in milliseconds, `10e3` by default
- `maxPendingBytes`: messages are rejected when more than this is waiting to be sent (e.g. the server is unreachable or does not read), `1 MiB` by default
A client uses at most one connection, opened when needed.
### `client.send(message, options)`
Returns a promise which resolves when the message has been handed to the system. Syslog has no acknowledgement: it does not mean that the server received it.
Options:
- `msgid`: message ID (RFC 5424 only)
- `severity`: one of `SEVERITIES` (e.g. `error`, `warning`, `informational`), `informational` by default
- `timestamp`: `Date` of the message, now by default
The promise rejects if the message cannot be sent, with the cause of the failure (e.g. connection refused, invalid certificate, too much data waiting).
### `client.close()`
Closes the connection, including a connection which is being established. The messages already sent are still delivered first (e.g. while the connection is being established or the host name resolved), for at most `closeTimeout`: after that, those not sent fail.

View File

@@ -1,36 +0,0 @@
.*
/benchmark/
/benchmarks/
*.bench.{,c,m}js
*.bench.{,c,m}js.map
/coverage/
/examples/
example.{,c,m}js
example.{,c,m}js.map
*.example.{,c,m}js
*.example.{,c,m}js.map
/fixture/
/fixtures/
*.fixture.{,c,m}js
*.fixture.{,c,m}js.map
*.fixtures.{,c,m}js
*.fixtures.{,c,m}js.map
/test/
/tests/
*.integ.{,c,m}js
*.integ.{,c,m}js.map
*.spec.{,c,m}js
*.spec.{,c,m}js.map
*.test.{,c,m}js
*.test.{,c,m}js.map
__snapshots__/
/docs/
/src/
eslint.config.mjs

View File

@@ -1,81 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/syslog
[![Package Version](https://badgen.net/npm/v/@vates/syslog)](https://npmjs.org/package/@vates/syslog) ![License](https://badgen.net/npm/license/@vates/syslog) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/syslog)](https://bundlephobia.com/result?p=@vates/syslog) [![Node compatibility](https://badgen.net/npm/node/@vates/syslog)](https://npmjs.org/package/@vates/syslog)
> Syslog client: RFC 5424 messages over UDP, TCP or TLS, with a bounded buffer
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/syslog):
```sh
npm install --save @vates/syslog
```
## Usage
```js
import { createClient } from '@vates/syslog'
const client = createClient('tls://syslog.company.lan', {
appName: 'my-app',
facility: 'authpriv',
})
await client.send('something happened', { msgid: 'event', severity: 'warning' })
// closes the connection once the messages already sent are delivered, the client reconnects if it is used again
client.close()
```
### `createClient(uri, options)`
`uri`: `udp://host[:port]`, `tcp://host[:port]` or `tls://host[:port]`, the default port is `514`, or `6514` for TLS.
Options:
- `appName`: application name of the messages, `node` by default
- `facility`: one of `FACILITIES` (e.g. `user`, `authpriv`, `local0`), `user` by default
- `format`: `rfc5424` (default) or `rfc3164`
- `framing` (TCP and TLS): `lf` (default, each message followed by a line feed, its own line breaks written as `\n` and `\r`) or `octet-counting` (each message prefixed by its length, required by some servers for TLS, see [RFC 5425](https://datatracker.ietf.org/doc/html/rfc5425#section-4.3))
- `hostname`: host name of the messages, `os.hostname()` by default
- `ca` (TLS): contents of PEM certificate authorities to trust, in addition to the system ones
- `allowUnauthorized` (TLS): do not check the certificate of the server, `false` by default
- `closeTimeout`: in milliseconds, how long `close()` waits for the messages already sent, `10e3` by default
- `connectTimeout`: in milliseconds, `10e3` by default
- `maxPendingBytes`: messages are rejected when more than this is waiting to be sent (e.g. the server is unreachable or does not read), `1 MiB` by default
A client uses at most one connection, opened when needed.
### `client.send(message, options)`
Returns a promise which resolves when the message has been handed to the system. Syslog has no acknowledgement: it does not mean that the server received it.
Options:
- `msgid`: message ID (RFC 5424 only)
- `severity`: one of `SEVERITIES` (e.g. `error`, `warning`, `informational`), `informational` by default
- `timestamp`: `Date` of the message, now by default
The promise rejects if the message cannot be sent, with the cause of the failure (e.g. connection refused, invalid certificate, too much data waiting).
### `client.close()`
Closes the connection, including a connection which is being established. The messages already sent are still delivered first (e.g. while the connection is being established or the host name resolved), for at most `closeTimeout`: after that, those not sent fail.
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,311 +0,0 @@
'use strict'
const dgram = require('dgram')
const net = require('net')
const os = require('os')
const splitHost = require('split-host')
const tls = require('tls')
// https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.1
const FACILITIES = {
__proto__: null,
kern: 0,
user: 1,
mail: 2,
daemon: 3,
auth: 4,
syslog: 5,
lpr: 6,
news: 7,
uucp: 8,
cron: 9,
authpriv: 10,
ftp: 11,
ntp: 12,
audit: 13,
alert: 14,
clock: 15,
local0: 16,
local1: 17,
local2: 18,
local3: 19,
local4: 20,
local5: 21,
local6: 22,
local7: 23,
}
exports.FACILITIES = Object.keys(FACILITIES)
const SEVERITIES = {
__proto__: null,
emergency: 0,
alert: 1,
critical: 2,
error: 3,
warning: 4,
notice: 5,
informational: 6,
debug: 7,
}
exports.SEVERITIES = Object.keys(SEVERITIES)
const DEFAULT_PORTS = {
tcp: 514,
// https://datatracker.ietf.org/doc/html/rfc5425#section-4.1
tls: 6514,
udp: 514,
}
// header fields must be printable ASCII characters without spaces
const toHeaderField = (value, maxLength) =>
String(value)
.replace(/[^\x21-\x7e]/g, '_')
.slice(0, maxLength) || '-'
const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
const pad2 = n => String(n).padStart(2, '0')
const FORMATTERS = {
// https://datatracker.ietf.org/doc/html/rfc5424#section-6
rfc5424: ({ appName, hostname, message, msgid, pri, timestamp }) =>
`<${pri}>1 ${timestamp.toISOString()} ${hostname} ${appName} ${process.pid} ${msgid} - ${message}`,
// https://datatracker.ietf.org/doc/html/rfc3164#section-4.1, local time without year
rfc3164: ({ appName, hostname, message, pri, timestamp }) =>
`<${pri}>${MONTHS[timestamp.getMonth()]} ${String(timestamp.getDate()).padStart(2, ' ')} ${pad2(timestamp.getHours())}:${pad2(timestamp.getMinutes())}:${pad2(timestamp.getSeconds())} ${hostname} ${appName}[${process.pid}]: ${message}`,
}
const FRAMERS = {
// https://datatracker.ietf.org/doc/html/rfc6587#section-3.4.2
//
// a line feed ends the message: the line breaks it contains are written as
// `\n` (and `\r`) so that it is received as one message, use octet-counting
// to keep them
lf: message => message.replace(/\r/g, '\\r').replace(/\n/g, '\\n') + '\n',
// https://datatracker.ietf.org/doc/html/rfc6587#section-3.4.1, required by RFC 5425 (TLS)
'octet-counting': message => `${Buffer.byteLength(message)} ${message}`,
}
const noop = Function.prototype
// connecting to a host name with several addresses (e.g. IPv4 and IPv6) fails
// with an AggregateError without message, use the messages of its errors
function withMessage(error) {
if (error.message === '' && Array.isArray(error.errors) && error.errors.length !== 0) {
error.message = error.errors.map(error => error.message).join(', ')
// the first line of the stack has been computed with the empty message
if (typeof error.stack === 'string') {
const i = error.stack.indexOf('\n')
error.stack = `${error.name}: ${error.message}${i === -1 ? '' : error.stack.slice(i)}`
}
}
return error
}
// UDP sockets: callbacks of the datagrams not yet sent (e.g. host name being
// resolved), the socket is closed once they are all called
const pendingDatagrams = new WeakMap()
function closeUdpSocket(socket, timeout) {
const pending = pendingDatagrams.get(socket)
if (pending.size === 0) {
socket.close()
return
}
pending.closing = true
// the datagrams which are still not sent then fail, so that no send() stays pending
setTimeout(() => {
if (pending.size !== 0) {
const callbacks = Array.from(pending)
pending.clear()
socket.close()
const error = new Error('the syslog client was closed before the message could be sent')
callbacks.forEach(cb => cb(error))
}
}, timeout).unref()
}
class SyslogClient {
constructor(
uri,
{
appName = 'node',
ca,
closeTimeout = 10e3,
connectTimeout = 10e3,
facility = 'user',
format = 'rfc5424',
framing = 'lf',
allowUnauthorized = false,
hostname = os.hostname(),
maxPendingBytes = 1024 * 1024,
} = {}
) {
const matches = /^(tcp|tls|udp):\/\/(.+)$/.exec(String(uri).trim())
const protocol = matches?.[1]
const { host, port = DEFAULT_PORTS[protocol] } = (matches !== null && splitHost(matches[2])) || {}
if (!host) {
throw new Error(
`invalid syslog URI: ${uri}, it should be like udp://host[:port], tcp://host[:port] or tls://host[:port]`
)
}
if (!(facility in FACILITIES)) {
throw new Error(`invalid syslog facility: ${facility}`)
}
if (!(format in FORMATTERS)) {
throw new Error(`invalid syslog format: ${format}`)
}
if (!(framing in FRAMERS)) {
throw new Error(`invalid syslog framing: ${framing}`)
}
this._appName = toHeaderField(appName, 48)
this._closeTimeout = closeTimeout
this._connectTimeout = connectTimeout
this._facility = FACILITIES[facility]
this._format = FORMATTERS[format]
this._frame = FRAMERS[framing]
this._host = host
this._hostname = toHeaderField(hostname, 255)
this._maxPendingBytes = maxPendingBytes
this._port = port
this._send = protocol === 'udp' ? this._sendUdp : this._sendStream
this._tlsOptions =
protocol === 'tls'
? {
// the `ca` option replaces the system certificate authorities
ca: ca === undefined ? undefined : [...tls.rootCertificates, ...ca],
rejectUnauthorized: !allowUnauthorized,
}
: undefined
// at most one socket at a time, created when needed
this._socket = undefined
}
// the messages already sent are still delivered (e.g. a connection which is
// being established, or a host name which is being resolved), then the socket
// is closed, at most closeTimeout later
//
// the client can still be used afterwards, it will reconnect
close() {
const socket = this._socket
if (socket === undefined) {
return
}
this._socket = undefined
if (socket instanceof net.Socket) {
// the pending writes are flushed before the connection is ended
socket.end()
const timeout = setTimeout(() => socket.destroy(), this._closeTimeout)
timeout.unref()
socket.once('close', () => clearTimeout(timeout))
} else {
closeUdpSocket(socket, this._closeTimeout)
}
}
// resolves when the message has been handed to the system, which does not
// mean that the server received it (syslog has no acknowledgement)
send(message, { msgid = '-', severity = 'informational', timestamp = new Date() } = {}) {
if (!(severity in SEVERITIES)) {
return Promise.reject(new Error(`invalid syslog severity: ${severity}`))
}
const formatted = this._format({
appName: this._appName,
hostname: this._hostname,
message: String(message),
msgid: toHeaderField(msgid, 32),
pri: this._facility * 8 + SEVERITIES[severity],
timestamp,
})
return new Promise((resolve, reject) => {
this._send(formatted, error => (error == null ? resolve() : reject(error)))
})
}
// TCP and TLS
_sendStream(message, cb) {
let socket = this._socket
if (socket === undefined) {
const tlsOptions = this._tlsOptions
const options = { host: this._host, port: this._port }
socket = this._socket =
tlsOptions === undefined ? net.connect(options) : tls.connect({ ...tlsOptions, ...options })
// a connection attempt that never completes (e.g. packets dropped by a
// firewall, or a TLS handshake without answer) must not stay around
//
// not socket.setTimeout() which is an inactivity timeout, postponed by
// each write during a TLS handshake
const connectTimeout = setTimeout(() => socket.destroy(new Error('connection timed out')), this._connectTimeout)
connectTimeout.unref()
socket.on(tlsOptions === undefined ? 'connect' : 'secureConnect', () => {
clearTimeout(connectTimeout)
// detect a server which disappeared without closing the connection
socket.setKeepAlive(true, 60e3)
})
// errors are passed to the callbacks of the pending writes
socket.on('error', noop)
socket.on('close', () => {
clearTimeout(connectTimeout)
if (this._socket === socket) {
this._socket = undefined
}
})
// do not prevent the process from stopping
socket.unref()
}
// messages are dropped when too much is waiting to be sent (e.g. the server
// is unreachable or no longer reads), instead of filling the memory, this
// also covers the messages written while the connection is being established
if (socket.writableLength > this._maxPendingBytes) {
cb(new Error('too much data waiting to be sent to the syslog server'))
return
}
// pending writes fail with a generic error when the connection fails, report its cause instead
socket.write(this._frame(message), error => cb(error == null ? error : withMessage(socket.errored ?? error)))
}
// one message per datagram, see https://datatracker.ietf.org/doc/html/rfc5426
_sendUdp(message, cb) {
let socket = this._socket
if (socket === undefined) {
socket = this._socket = dgram.createSocket(net.isIPv6(this._host) ? 'udp6' : 'udp4')
pendingDatagrams.set(socket, new Set())
// errors are passed to the callbacks of send(), a failed socket is replaced
socket.on('error', () => {
if (this._socket === socket) {
this.close()
}
})
socket.unref()
}
const pending = pendingDatagrams.get(socket)
pending.add(cb)
socket.send(message, this._port, this._host, error => {
// otherwise, already failed by closeUdpSocket()
if (pending.delete(cb)) {
if (pending.size === 0 && pending.closing) {
socket.close()
}
cb(error)
}
})
}
}
exports.createClient = (uri, opts) => new SyslogClient(uri, opts)

View File

@@ -1,286 +0,0 @@
import assert from 'node:assert/strict'
import dgram from 'node:dgram'
import fs from 'node:fs'
import net from 'node:net'
import tls from 'node:tls'
import { after, describe, it } from 'node:test'
import { setTimeout as wait } from 'node:timers/promises'
import { createClient, FACILITIES, SEVERITIES } from './index.js'
const fixture = name => fs.readFileSync(new URL(`./tests/${name}`, import.meta.url), 'utf8')
const CA = fixture('ca.pem')
const servers = []
after(() => servers.forEach(server => server.close()))
const listen = server =>
new Promise(resolve => {
servers.push(server)
server.listen(0, '127.0.0.1', () => resolve(server.address().port))
})
// stream server collecting the received data
async function createStreamServer(createServer = net.createServer) {
const server = { data: '', sockets: new Set() }
server.port = await listen(
createServer(socket => {
server.sockets.add(socket)
socket.on('close', () => server.sockets.delete(socket))
socket.on('data', data => (server.data += data))
})
)
return server
}
async function createUdpServer() {
const messages = []
const socket = dgram.createSocket('udp4').on('message', message => messages.push(message.toString()))
servers.push(socket)
await new Promise(resolve => socket.bind(0, '127.0.0.1', resolve))
return { messages, port: socket.address().port }
}
describe('createClient()', () => {
it('rejects invalid URIs and options', () => {
for (const uri of ['syslog.lan', 'http://syslog.lan', 'udp://', 'udp://:514', undefined]) {
assert.throws(() => createClient(uri), /invalid syslog URI/, String(uri))
}
assert.throws(() => createClient('udp://h', { facility: 'foo' }), /invalid syslog facility/)
assert.throws(() => createClient('udp://h', { format: 'foo' }), /invalid syslog format/)
assert.throws(() => createClient('udp://h', { framing: 'foo' }), /invalid syslog framing/)
})
it('exposes the facility and severity names', () => {
assert(FACILITIES.includes('authpriv'))
assert.equal(SEVERITIES[4], 'warning')
})
})
describe('send()', () => {
it('formats RFC 5424 messages', async () => {
const server = await createUdpServer()
const client = createClient(` udp://127.0.0.1:${server.port} `, {
appName: 'my app',
facility: 'authpriv',
hostname: 'host',
})
await client.send('{"foo":"bar"}', {
msgid: 'some event',
severity: 'warning',
timestamp: new Date('2026-01-02T03:04:05.678Z'),
})
await wait(50)
client.close()
// header fields are sanitized: no spaces
assert.deepEqual(server.messages, [
`<84>1 2026-01-02T03:04:05.678Z host my_app ${process.pid} some_event - {"foo":"bar"}`,
])
})
it('formats RFC 3164 messages', async () => {
const server = await createUdpServer()
const client = createClient(`udp://127.0.0.1:${server.port}`, {
appName: 'app',
format: 'rfc3164',
hostname: 'host',
})
await client.send('message', { timestamp: new Date(2026, 0, 2, 3, 4, 5) })
await wait(50)
client.close()
assert.deepEqual(server.messages, [`<14>Jan 2 03:04:05 host app[${process.pid}]: message`])
})
it('rejects an invalid severity', async () => {
await assert.rejects(
createClient('udp://127.0.0.1').send('message', { severity: 'foo' }),
/invalid syslog severity/
)
})
it('frames TCP messages with a line feed by default', async () => {
const server = await createStreamServer()
const client = createClient(`tcp://127.0.0.1:${server.port}`)
await client.send('foo')
await client.send('bar')
await wait(50)
client.close()
assert.match(server.data, /^<14>1 [^\n]+ - foo\n<14>1 [^\n]+ - bar\n$/)
})
it('escapes the line breaks of a message framed with a line feed', async () => {
const server = await createStreamServer()
const client = createClient(`tcp://127.0.0.1:${server.port}`)
await client.send('foo\r\nbar\nbaz')
await wait(50)
client.close()
assert.match(server.data, /^<14>1 [^\n]+ - foo\\r\\nbar\\nbaz\n$/)
})
it('frames TCP messages with octet counting', async () => {
const server = await createStreamServer()
const client = createClient(`tcp://127.0.0.1:${server.port}`, { framing: 'octet-counting' })
await client.send('foo')
await client.send('b\när')
await wait(50)
client.close()
// each message is prefixed by its length in bytes
const buffer = Buffer.from(server.data)
const messages = []
for (let offset = 0; offset < buffer.length; ) {
const space = buffer.indexOf(' ', offset)
const length = Number(buffer.toString('utf8', offset, space))
messages.push(buffer.toString('utf8', space + 1, space + 1 + length))
offset = space + 1 + length
}
assert.equal(messages.length, 2)
assert.match(messages[0], /^<14>1 .+ - foo$/)
assert.match(messages[1], /^<14>1 .+ - b\när$/s)
})
it('reconnects when the server closes the connection', async () => {
const server = await createStreamServer()
const client = createClient(`tcp://127.0.0.1:${server.port}`)
await client.send('first')
await wait(50)
server.sockets.forEach(socket => socket.destroy())
await wait(50)
await client.send('second')
await wait(50)
client.close()
assert.match(server.data, /first\n.*second\n$/s)
})
it('reports why the connection failed', async () => {
await assert.rejects(createClient('tcp://127.0.0.1:1').send('message'), { code: 'ECONNREFUSED' })
// several addresses (::1 and 127.0.0.1) fail with an AggregateError, which has no message by itself
await assert.rejects(createClient('tcp://localhost:1').send('message'), error => {
assert.match(error.message, /ECONNREFUSED/)
assert(error.stack.startsWith(`${error.name}: ${error.message}\n`))
return true
})
})
it('stops connecting after connectTimeout, with a single socket', async () => {
// accepts the TCP connection but never answers the TLS handshake
const port = await listen(net.createServer(socket => socket.resume()))
const client = createClient(`tls://127.0.0.1:${port}`, { connectTimeout: 200 })
const sockets = new Set()
const start = Date.now()
const result = client.send('message').catch(error => error)
sockets.add(client._socket)
client.send('message').catch(Function.prototype)
sockets.add(client._socket)
assert.equal((await result).message, 'connection timed out')
assert(Date.now() - start < 1e3)
assert.equal(sockets.size, 1)
})
it('drops messages when too much is waiting to be sent', async () => {
// never reads
const port = await listen(net.createServer(socket => socket.pause()))
const client = createClient(`tcp://127.0.0.1:${port}`, { closeTimeout: 200, maxPendingBytes: 64 * 1024 })
const message = 'x'.repeat(1024)
const results = []
for (let i = 0; i < 10e3; ++i) {
results.push(
client.send(message).then(
() => 'sent',
() => 'dropped'
)
)
}
await wait(500)
client.close()
assert(client._socket === undefined)
assert((await Promise.all(results)).includes('dropped'))
})
it('delivers the messages of a connection which is being established, then closes it', async () => {
const server = await createStreamServer()
const client = createClient(`tcp://127.0.0.1:${server.port}`)
const sent = client.send('message')
const socket = client._socket
client.close()
await sent
await new Promise(resolve => (socket.closed ? resolve() : socket.once('close', resolve)))
assert.match(server.data, /message\n$/)
assert(socket.destroyed)
assert.equal(client._socket, undefined)
})
it('delivers the datagrams whose host name is being resolved, then closes the socket', async () => {
const server = await createUdpServer()
const client = createClient(`udp://localhost:${server.port}`)
const sent = client.send('message')
client.close()
await sent
await wait(100)
assert.equal(server.messages.length, 1)
assert.match(server.messages[0], /message$/)
})
it('stops waiting for the pending writes after closeTimeout', async () => {
// never reads
const port = await listen(net.createServer(socket => socket.pause()))
const client = createClient(`tcp://127.0.0.1:${port}`, { closeTimeout: 200, maxPendingBytes: Infinity })
const results = []
for (let i = 0; i < 1e3; ++i) {
results.push(client.send('x'.repeat(64 * 1024)).then(Function.prototype, error => error))
}
const socket = client._socket
client.close()
const start = Date.now()
await Promise.all(results)
assert(Date.now() - start < 2e3)
assert(socket.destroyed)
})
})
describe('TLS', () => {
const createTlsServer = () =>
createStreamServer(listener =>
tls.createServer({ cert: fixture('server.pem'), key: fixture('server.key') }, listener)
)
it('trusts additional certificate authorities', async () => {
const server = await createTlsServer()
const client = createClient(`tls://localhost:${server.port}`, { ca: [CA] })
await client.send('message')
await wait(50)
client.close()
assert.match(server.data, / - message\n$/)
})
it('rejects an unknown certificate authority, unless allowUnauthorized', async () => {
const server = await createTlsServer()
await assert.rejects(createClient(`tls://localhost:${server.port}`).send('message'), {
code: 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
})
const client = createClient(`tls://localhost:${server.port}`, { allowUnauthorized: true })
await client.send('message')
client.close()
})
it('defaults to port 6514', () => {
assert.equal(createClient('tls://localhost')._port, 6514)
assert.equal(createClient('tcp://[::1]')._port, 514)
assert.equal(createClient('tcp://[::1]')._host, '::1')
})
})

View File

@@ -1,33 +0,0 @@
{
"private": false,
"name": "@vates/syslog",
"description": "Syslog client: RFC 5424 messages over UDP, TCP or TLS, with a bounded buffer",
"keywords": [
"rfc5424",
"syslog",
"tls"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/syslog",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/syslog",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.1.0",
"engines": {
"node": ">=18"
},
"dependencies": {
"split-host": "^2.0.0"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node --test"
}
}

View File

@@ -1,19 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDITCCAgmgAwIBAgIUGikUHbKIfotmlkV7oGy7YubSDn8wDQYJKoZIhvcNAQEL
BQAwHzEdMBsGA1UEAwwUdmF0ZXMtc3lzbG9nIHRlc3QgQ0EwIBcNMjYxMDE4MTAx
MjI1WhgPMjEyNjA5MjQxMDEyMjVaMB8xHTAbBgNVBAMMFHZhdGVzLXN5c2xvZyB0
ZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn/lGtdLEE8tf
gb+hSPe8xLOlGh7ZzRiGcKjkcCbCNFNnjjOa29Lki8ksFD7Su2BWIHpxAzPhEnlQ
9xq7+M9dayM2CslQPuOfKEEyQIBWG9OggXT3kyTn0BVHNFfiGzJlyRz6fIfLf/sU
e2d9AWZhP9I53HpKaNMHyxwuzEBWFxmpv1t7LRVwhrcQjsnoZqxlukpsGyKkT4gs
TNH5/hFhZzmnvfFPhTzQvOGSvFQqgUgd5uxidDmImoRoGA9Ip3bqwMkT99euNrgc
jsxnXeV8pdtHmz4kjQRp+/YIIoh7pA73389OeBx7LC9wpZo1z+3AVKlCAGZ8V9Il
Izk3e7FXTQIDAQABo1MwUTAdBgNVHQ4EFgQUTx7dqoC2XIwpwW0MYcZsoXHqztww
HwYDVR0jBBgwFoAUTx7dqoC2XIwpwW0MYcZsoXHqztwwDwYDVR0TAQH/BAUwAwEB
/zANBgkqhkiG9w0BAQsFAAOCAQEARYUd0q9VwgmZGdhhqks1xCipBuW1eWwBa0jk
Y0jZjhxeq9F0KBMVoazRjjKoL6cwpjvUKytarnN8+kj/7U4dMPLX+/bcpEjwo1gW
DI7MtBI+e1+IohWujDtCqyAiPYpWlwlTTC4kMdfjDH4fbYNY6UHUnedPdSwN3HAp
ZtMb1RoT+gtMedpbnQJKTwxAwH5cBMjTgKeuxeHo6WWjB6evyaLjQINNGLDgLxUC
i4+o0rNYCPtJV3uqw9egDkgoAtMI9mY6snMUxaHGz2JhGzT1wu/UXPrkbFYxCYpm
BGxY36taw81Mr+6n0V1ScWhnGEgPOe3BJCYtoB964YDYfvcFLA==
-----END CERTIFICATE-----

View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCL0wuMNV3AkIhP
gt4eQ9IPbOkbZiyNOgQWanaRr1P5XeEz4wbWH03MNf9gUDbR2ToGJLfxU68upFYo
uQRCRHRySdbzvhc5+OYbQ2/QvucGKaMnYuerxvENgnh2eLg5QG+Z50M8+IK+jK4t
YJjECe5nefjdtGKQf1h4xgVJp2CQW2zKyfOBqjgcB6bOISDni9+1ssWDkcgVnqyB
8o/zA11CFfO1AL0OBbpD6Fk9HZ29hODCQLOq01vIrfpV2H8S/hsik4tUtMgM10y5
7LNckDzd8LbZQiBUJ/Q63pWDN19SqAW9lsonE3aKJANvew2s/NRtU7EKIeKLvf3a
HPJH83QBAgMBAAECggEABrXXOphrTlj1XL0A5kqOkchSF9s3gUPW80Ey3Wcg2GFo
s9f18ZfdIMM45T02Q0wJtoajuAKCUFKJUsEd1uUQglVgOjB9zn2/I9DwxF+sCEn+
Yy8SQ5Z7pjCEMdnVbAR3UqdUTRBXXHgVihaWIEbifM9wCcpM1mbRJvMoaO5uIkFZ
ztlYElhFC+ITpm8Pjs/fQPNlA8DPZO3xHRSx+M0+tNw3ba7SaFvYTA2o/sMOx1Yq
pyWf7yq6kPDE3vO6NfO8uYrfkw/eNioQ3jZh9tlH/87hm1ucXFOHKxdCpabv2COJ
YIOqIcXlRbXlzVugDY6IOdBcCvUM6DdB+iyOx2m6UQKBgQDEyHoCCJZhLf7ynsmu
onmonWGojX6EwDgpvoey2Lhbr9j7gkCkx7TF+CNRN4AM4o+K9x+m8rF2aJEXKgof
GNwYPpBaUwuV2y+A9wiYtM+o9D4pTy2JPzlfyj7zWIQtOzrIkgSz9BH7QYRA51it
L0d9O6myUs2IILcXdb7gTePA0QKBgQC15qgJ4DeDq+VygQi1VW6Xn2DhiAy1frDY
etG3We+PIKYOc9XN79hI7cSXAy9DYwX7KTQJ13YDYDKNuEEGlijatxMruJv2Zvdm
5CfgNzw+QjFfOM0AXsp9/F4xiiSiUAvLJeZzN7BVPQ/7KUz6kMxtr8RorP66NSFt
zplrq6XMMQKBgQCP+EyraDT9bHV2dI4B4oU3w81LUBocbbWpdOg7BSv8iK8rc6Rv
6lxr6puMNBrPUoLKnPrfXfhQNTjRxcQZ9EoywrNCIjf7r50Vsabscnzrk6ZW+sLK
AyOq3YEo8L3jic38KDigoBT3yntEMP5D1q9rBFenS8VrruPFmjPIT7QtwQKBgQCd
To/bmo/nyuxujbBeQTeW4DoEUIfNHXcKf4yFp71IZc+1h+9KXTUKPLVac2KYQDaX
8hYSvMZOPQ77CHETpSBHpTNrCAe4m2sxC/aALMRogw33b0O6Ij+znGBJL1JnHBpW
4LvewDY6QMhlSF5OApMH+I0+iPzxQ6lZMOX0+0gywQKBgEGcCeJBvqpnSOyMdOFu
MIcLhGa2SIttij1jz7eS9zD+VeONHjmn3Atn7MRagddrYeIGy6ZHUFFsSWBbVqwB
cfT6o0GOeNx0gQn185SUBxB3Oxfg4xWBA0q0GVNFqaScKiXk1IvTj/GTuXM7rehi
HjK6iamDr8lIJE/JFdn2K8ge
-----END PRIVATE KEY-----

View File

@@ -1,20 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDMzCCAhugAwIBAgIUSGpRtJFiD61hQZBOp49sq7GLkIswDQYJKoZIhvcNAQEL
BQAwHzEdMBsGA1UEAwwUdmF0ZXMtc3lzbG9nIHRlc3QgQ0EwIBcNMjYxMDE4MTAx
MjI1WhgPMjEyNjA5MjQxMDEyMjVaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIw
DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIvTC4w1XcCQiE+C3h5D0g9s6Rtm
LI06BBZqdpGvU/ld4TPjBtYfTcw1/2BQNtHZOgYkt/FTry6kVii5BEJEdHJJ1vO+
Fzn45htDb9C+5wYpoydi56vG8Q2CeHZ4uDlAb5nnQzz4gr6Mri1gmMQJ7md5+N20
YpB/WHjGBUmnYJBbbMrJ84GqOBwHps4hIOeL37WyxYORyBWerIHyj/MDXUIV87UA
vQ4FukPoWT0dnb2E4MJAs6rTW8it+lXYfxL+GyKTi1S0yAzXTLnss1yQPN3wttlC
IFQn9DrelYM3X1KoBb2WyicTdookA297Daz81G1TsQoh4ou9/doc8kfzdAECAwEA
AaNwMG4wLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA
AAABMB0GA1UdDgQWBBTCMy+a1aPNwrF3QYPIAIR4iDazWTAfBgNVHSMEGDAWgBRP
Ht2qgLZcjCnBbQxhxmyhcerO3DANBgkqhkiG9w0BAQsFAAOCAQEAPWBHmtiuPwFb
z0DIdTOA7VvFziMyJzW0EvI2AmB2wSViPGg5OFqVOLq5+H3OJsi5yTYJaUjNpr4A
UjpMweJJlMkqwsTt/kBKpPiUQbiN7Ja2+b3mmmMhEfyMUSebcfegLNvglABhJlmv
FOwniX8J1LJ8fzxAHfF32HwqxG6vxcZOsP661WqSnkyDSuoHo/jwB4MeCZiljc8h
F4MkcaIGouTtAGMvb8V6bGNz1iWgkQVpfYP9uqKGq8eBXzd+/033RmrnqorHjYiz
38C1Gf3lhbXBQqmMY3+aEhvjSCJrLCfjMV/twDTwRp8khrkG7C9yuEqmXtbTcF1e
/+rsiTEv9A==
-----END CERTIFICATE-----

View File

@@ -11,10 +11,6 @@
"./xo": "./dist/xo.d.mts",
"./xo-app": "./dist/xo-app.d.mts",
"./xen-api": "./dist/xen-api.d.mts",
"./backup-expression-context": {
"default": "./dist/backup-expression-context.mjs",
"types": "./dist/backup-expression-context.d.mts"
},
"./common": {
"default": "./dist/common.mjs",
"types": "./dist/common.d.mts"

View File

@@ -1,38 +0,0 @@
export const BACKUP_EXPRESSION_CONTEXT = {
run: {
dayOfWeek: { type: 'number' },
dayOfMonth: { type: 'number' },
hour: { type: 'number' },
month: { type: 'number' },
year: { type: 'number' },
},
sr: {
name_label: { type: 'string' },
type: { type: 'string' },
description: { type: 'string' },
tags: { type: 'string[]' },
},
backupRepository: {
name: { type: 'string' },
type: { type: 'string' },
tags: { type: 'string[]' },
},
vm: {
name_label: { type: 'string' },
tags: { type: 'string[]' },
power_state: { type: 'string' },
},
chainLength: { type: 'number' },
} as const
export type BACKUP_EXPRESSION_CONTEXT = (typeof BACKUP_EXPRESSION_CONTEXT)[keyof typeof BACKUP_EXPRESSION_CONTEXT]
export const BACKUP_SETTING_CONTEXT = {
healthCheckVmsWithTags: ['vm', 'run'],
fullInterval: ['vm', 'run', 'chainLength'],
vmFilter: ['vm', 'run'],
srs: ['sr', 'run'],
remotes: ['backupRepository', 'run'],
} as const
export type BACKUP_SETTING_CONTEXT = (typeof BACKUP_SETTING_CONTEXT)[keyof typeof BACKUP_SETTING_CONTEXT]

View File

@@ -772,7 +772,6 @@ export type RawTrafficRule = {
ipRange: string
direction: TrafficRuleDirection
port?: string
priority?: number
}
type BaseTrafficRule = RawTrafficRule & {

View File

@@ -5,7 +5,5 @@ export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/xen-orchestra-acl.mjs'
export * from './lib/xen-orchestra-plugins.mjs'
export * from './lib/xen-orchestra-v2v.mjs'
export * from './lib/complex-matcher.mjs'
export * from './lib/rest-api.mjs'
export * from './lib/xen-orchestra-vmware-explorer.mjs'

View File

@@ -1,221 +0,0 @@
import type { Branded } from '../common.mjs'
import type { XoNetwork, XoSr, XoVmTemplate } from '../xo.mjs'
// Stateless API: requests needing the source carry the connection in a POST body (never in URLs or access logs) and
// xo-server stores nothing it reads from the source.
export const V2V_PLATFORM = {
VMWARE: 'vmware',
} as const
export type V2V_PLATFORM = (typeof V2V_PLATFORM)[keyof typeof V2V_PLATFORM]
// 422 rather than 401 for invalid source credentials: a 401 would open the browser Basic authentication prompt
export const V2V_ERROR_CODE = {
INVALID_CREDENTIALS: 'V2V_INVALID_CREDENTIALS',
} as const
export type V2V_ERROR_CODE = (typeof V2V_ERROR_CODE)[keyof typeof V2V_ERROR_CODE]
// Clones, restores and replicas inherit `other_config`: the imported VM records its own UUID to be told apart from
// them, and the source identity because a moref is only unique within one vCenter. The source identity is the
// inventory `instanceUuid`, or `<lowercased host>:<port>` for a standalone ESXi which has none.
export const V2V_VM_OTHER_CONFIG = {
SOURCE_VM_ID: 'sourceVmId',
SOURCE_SNAPSHOT_ID: 'sourceSnapshotId',
IMPORTED_VM_UUID: 'xo:v2v:importedVmUuid',
SOURCE_ID: 'xo:v2v:sourceId',
SOURCE_VM_INSTANCE_UUID: 'xo:v2v:sourceVmInstanceUuid',
} as const
export type V2V_VM_OTHER_CONFIG = (typeof V2V_VM_OTHER_CONFIG)[keyof typeof V2V_VM_OTHER_CONFIG]
export type XoV2vVmOtherConfig = Partial<Record<V2V_VM_OTHER_CONFIG, string>>
// The disk of the source chain an imported VDI last received, one of the `chainPaths` of its source disk: the CID
// cannot tell a fresh snapshot delta from its parent, the path can
export const V2V_VDI_OTHER_CONFIG = {
DISK_PATH: 'esxi_diskPath',
} as const
export type V2V_VDI_OTHER_CONFIG = (typeof V2V_VDI_OTHER_CONFIG)[keyof typeof V2V_VDI_OTHER_CONFIG]
export const V2V_START_BLOCKED_REASON = 'Esxi migration in progress...'
export const V2V_PARTIAL_IMPORT_TAG = 'xo:no-bak=V2V partial import'
export const V2V_SNAPSHOT_NAME = {
AFTER_PARTIAL_IMPORT: 'after partial import from V2V',
AFTER_COMPLETE_IMPORT: 'after complete import from V2V',
COMPLETE_IMPORT: 'complete import from V2V',
} as const
export type V2V_SNAPSHOT_NAME = (typeof V2V_SNAPSHOT_NAME)[keyof typeof V2V_SNAPSHOT_NAME]
// Set on the task of a start so the front can tell which source VMs are being imported
export const V2V_TASK_PROPERTY = {
SOURCE_VM_IDS: 'sourceVmIds',
} as const
export type XoV2vConnection = {
host: string
port: number
username: string
password: string
skipSslCheck: boolean
}
export type XoV2vStatus = {
vectura: { status: 'success' } | { status: 'error'; error: string }
}
export type XoV2vInventoryPayload = {
platform: V2V_PLATFORM
connection: XoV2vConnection
}
export type XoV2vDatacenter = {
id: Branded<'v2v-datacenter'>
name: string
}
export type XoV2vCluster = {
id: Branded<'v2v-cluster'>
name: string
datacenterId: XoV2vDatacenter['id']
}
export type XoV2vHost = {
id: Branded<'v2v-host'>
name: string
clusterId?: XoV2vCluster['id']
datacenterId: XoV2vDatacenter['id']
}
// Sizes are in bytes, as in the rest of XO, not in the KB/MB VMware returns
export type XoV2vDatastore = {
id: Branded<'v2v-datastore'>
name: string
type: string
capacity: number
free: number
hostIds: XoV2vHost['id'][]
}
export type XoV2vPortGroup = {
id: Branded<'v2v-port-group'>
name: string
vlan?: number
}
export const V2V_DISK_MODE = {
PERSISTENT: 'persistent',
NON_PERSISTENT: 'nonpersistent',
INDEPENDENT_PERSISTENT: 'independent_persistent',
INDEPENDENT_NON_PERSISTENT: 'independent_nonpersistent',
UNDOABLE: 'undoable',
APPEND: 'append',
} as const
export type V2V_DISK_MODE = (typeof V2V_DISK_MODE)[keyof typeof V2V_DISK_MODE]
// `key` is the vmx device node (`scsi0:0`, `ethernet0`…), the identifier the import matches the start maps against
export type XoV2vDisk = {
key: Branded<'v2v-disk'>
label: string
datastoreId: XoV2vDatastore['id']
capacity: number
mode: V2V_DISK_MODE
isRdm: boolean
isEncrypted: boolean
// `dir/disk.vmdk` in its datastore (no `[datastore]` prefix) of the base disk, then of each snapshot delta down to
// the active disk: the chain the import walks for this vmx node
chainPaths: string[]
}
// No port group for opaque (NSX) or missing backings
export type XoV2vNic = {
key: Branded<'v2v-nic'>
label: string
portGroupId?: XoV2vPortGroup['id']
macAddress: string
}
export const V2V_VM_POWER_STATE = {
POWERED_ON: 'poweredOn',
POWERED_OFF: 'poweredOff',
SUSPENDED: 'suspended',
} as const
export type V2V_VM_POWER_STATE = (typeof V2V_VM_POWER_STATE)[keyof typeof V2V_VM_POWER_STATE]
export const V2V_FIRMWARE = {
BIOS: 'bios',
UEFI: 'uefi',
} as const
export type V2V_FIRMWARE = (typeof V2V_FIRMWARE)[keyof typeof V2V_FIRMWARE]
export type XoV2vVm = {
id: Branded<'v2v-vm'>
instanceUuid: string
name: string
description: string
powerState: V2V_VM_POWER_STATE
hostId: XoV2vHost['id']
nCpus: number
memory: number
firmware: V2V_FIRMWARE
secureBoot: boolean
datastoreIds: XoV2vDatastore['id'][]
snapshotCount: number
// Compared with `other_config.sourceSnapshotId` of a partial import: a delta is only possible while they match
currentSnapshotId?: string
disks: XoV2vDisk[]
nics: XoV2vNic[]
}
export type XoV2vInventory = {
product: string
version: string
// Absent on a standalone ESXi, identified by its connection address instead
instanceUuid?: string
datacenters: XoV2vDatacenter[]
clusters: XoV2vCluster[]
hosts: XoV2vHost[]
datastores: XoV2vDatastore[]
portGroups: XoV2vPortGroup[]
vms: XoV2vVm[]
}
export type XoV2vStartOptions = {
stopSource: boolean
// once the import is complete, start the VM, following the first boot of a docked guest
startVm: boolean
// once the import is complete, prepare the guest to boot on XCP-ng with the docking ISO of the pool, off by default
docking?: boolean
}
// A resumed import sends no disk or NIC map: the back end reads its mapping from the existing XCP-ng VM
export type XoV2vStartPayloadVm = Partial<XoV2vStartOptions> & {
id: XoV2vVm['id']
disks?: Record<XoV2vDisk['key'], XoSr['id']>
nics?: Record<XoV2vNic['key'], XoNetwork['id']>
template?: XoVmTemplate['id']
tags: string[]
keepMac: boolean
}
// Body of `POST /pools/{id}/actions/import_vms_from_vmware`: the pool is in the path. Checked against the body of the
// route when the REST API is built, which refuses any extra key
export type XoV2vStartPayload = {
connection: XoV2vConnection
defaults: XoV2vStartOptions & {
sr: XoSr['id']
network: XoNetwork['id']
template: XoVmTemplate['id']
}
vms: XoV2vStartPayloadVm[]
stopOnError: boolean
concurrency: number
}

View File

@@ -1,105 +0,0 @@
// shape of `Esxi#getInventory()` in `@xen-orchestra/vmware-explorer`
// the objects reference each other through their VMware managed object references (moref)
export type VmwareDatacenter = {
id: string
name: string
}
export type VmwareCluster = {
id: string
name: string
datacenter?: string
}
export type VmwareHost = {
id: string
name: string
/** a cluster, or the compute resource of a standalone host */
parent?: string
datacenter?: string
}
export type VmwareDatastore = {
id: string
name?: string
type?: string
capacity?: number
freeSpace?: number
/** the hosts which mount the datastore and can reach it */
hosts: string[]
}
export type VmwareVlan = number | { start: number; end: number }[]
export type VmwareStandardNetwork = {
id: string
name: string
distributed: false
}
export type VmwareDistributedPortgroup = {
id: string
name: string
distributed: true
vlan?: VmwareVlan
}
export type VmwareNetwork = VmwareStandardNetwork | VmwareDistributedPortgroup
export type VmwareVmDisk = {
/** node of the disk, as the vmx names it, e.g. `scsi0:0` */
key: string
label: string
capacity: number
datastore?: string
diskMode?: string
rdm: boolean
encrypted: boolean
/** `dir/disk.vmdk` in its datastore of the base disk, then of each snapshot delta down to the active disk */
chainPaths: string[]
}
export type VmwareVmNic = {
/** device key of the card */
key: string
label: string
macAddress?: string
addressType?: string
network?: string
/** what the predictable name of the card in the guest derives from (192 → `ens192`) */
pciSlotNumber?: number
}
export type VmwareVm = {
id: string
name?: string
annotation?: string
instanceUuid?: string
firmware?: string
secureBoot: boolean
isTemplate: boolean
cpus?: number
memory?: number
powerState?: 'poweredOff' | 'poweredOn' | 'suspended'
host?: string
datastores: string[]
currentSnapshot?: string
snapshotCount: number
disks: VmwareVmDisk[]
nics: VmwareVmNic[]
}
export type VmwareInventory = {
/** e.g. `VMware vCenter Server`, `VMware ESXi` */
product?: string
version?: string
/** only set by a vCenter */
instanceUuid?: string
datacenters: VmwareDatacenter[]
clusters: VmwareCluster[]
hosts: VmwareHost[]
datastores: VmwareDatastore[]
networks: VmwareNetwork[]
vms: VmwareVm[]
}

View File

@@ -3,17 +3,12 @@ import type {
AnyXoJob,
AnyXoLog,
NonXapiXoRecord,
XoBackupDiskPartition,
XapiXoRecord,
XoAuthenticationToken,
XoBackupRepository,
XoBrowserMediaSession,
XoConfigBackupArchive,
XoGroup,
XoHost,
XoMetadataBackupJob,
XoMirrorBackupJob,
XoNetwork,
XoPlugin,
XoPool,
XoPoolBackupArchive,
@@ -23,17 +18,12 @@ import type {
XoSr,
XoTask,
XoUser,
XoVdi,
XoVif,
XoVm,
XoVmBackupArchive,
XoVmBackupJob,
XoVmTemplate,
} from './xo.mjs'
import { VatesTask } from './lib/vates-task.mjs'
import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
import type { VmwareInventory } from './lib/xen-orchestra-vmware-explorer.mjs'
import type { XoV2vStatus } from './lib/xen-orchestra-v2v.mjs'
import type { RPU_RECOVERY_STEP_NAME } from './common.mjs'
import {
Xapi,
@@ -212,15 +202,6 @@ type XoStoreRangeOptions = {
reverse?: boolean
}
// `@xen-orchestra/vmware-explorer` is not typed
export type EsxiConnectionParams = {
/** `host` or `host:port` */
host: string
user: string
password: string
sslVerify: boolean
}
export type XoApp = EventEmitter & {
hooks: EventEmitter
_redis: {
@@ -301,8 +282,6 @@ export type XoApp = EventEmitter & {
backupGuard(objectId: XapiXoRecord['id'], opts?: { bypassBackupCheck?: boolean; operation: string }): Promise<void>
/* Throw if no authorization */
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
/* whether this machine can run the tools of an import from VMware */
checkEsxiDependencies(): Promise<XoV2vStatus>
/* validate, apply and persist the configuration of a plugin */
configurePlugin(id: string, configuration: unknown, mergeWithExisting?: boolean): Promise<void>
/* connect a server (XCP-ng/XenServer) */
@@ -330,17 +309,6 @@ export type XoApp = EventEmitter & {
set(key: string, value: unknown): void
unset(key: string): void
}
createBackupNgJob<T extends XoVmBackupJob | XoMirrorBackupJob>(
type: T['type'],
props: Omit<T, 'id' | 'type'>,
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
schedules?: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
): Promise<T>
createMetadataBackupJob(
props: Omit<XoMetadataBackupJob, 'id' | 'type'>,
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
schedules: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
): Promise<XoMetadataBackupJob>
createAuthenticationToken(opts: {
client?: {
id?: string
@@ -356,14 +324,6 @@ export type XoApp = EventEmitter & {
proxy?: XoProxy['id']
url: string
}): Promise<XoBackupRepository>
createSchedule(params: {
cron: XoSchedule['cron']
enabled: XoSchedule['enabled']
jobId: XoSchedule['jobId']
name?: XoSchedule['name']
timezone?: XoSchedule['timezone']
userId?: XoUser['id']
}): Promise<XoSchedule>
createUser(params: {
firstname?: string
lastname?: string
@@ -380,7 +340,6 @@ export type XoApp = EventEmitter & {
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteSchedule(id: XoSchedule['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
detachHostFromPool(hostId: XoHost['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
@@ -419,8 +378,6 @@ export type XoApp = EventEmitter & {
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
getAllRemotes(): Promise<XoBackupRepository[]>
removeRemote(id: XoBackupRepository['id']): Promise<void>
/** Also deletes the schedules of the job */
removeJob(id: AnyXoJob['id']): Promise<void>
getAllRemotesInfo(): Promise<
Record<
XoBackupRepository['id'],
@@ -437,23 +394,6 @@ export type XoApp = EventEmitter & {
>
>
getAllSchedules(): Promise<XoSchedule[]>
/** Every browser media session, whoever owns it; empty when the feature is disabled */
getAllBrowserMediaSessions(): XoBrowserMediaSession[]
getBrowserMediaSession(id: XoBrowserMediaSession['id']): XoBrowserMediaSession
/**
* Insert the ISO of a browser media session in the CD drive of a VM. Undone by `ejectBrowserMedia`,
* or when the browser disconnects.
*
* `assertBrowserMediaInsertable` reports the predictable failures synchronously, e.g. before
* starting a task.
*/
insertBrowserMedia(params: {
id: XoBrowserMediaSession['id']
vmId: XoVm['id']
}): Promise<{ vdiId: XoVdi['id'] }>
assertBrowserMediaInsertable(params: { id: XoBrowserMediaSession['id']; vmId: XoVm['id'] }): void
/** Eject the ISO of a browser media session, release its storage, and disconnect the browser */
ejectBrowserMedia(id: XoBrowserMediaSession['id']): Promise<void>
getAllUsers(opts?: { obfuscatePassword?: boolean }): Promise<XoUser[]>
getAllXenServers(): Promise<XoServer[]>
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
@@ -501,7 +441,6 @@ export type XoApp = EventEmitter & {
}>
getUser: (id: XoUser['id'], opts?: { obfuscatePassword?: boolean }) => Promise<XoUser>
getUserIdentityFields(): string[]
getEsxiInventory(params: EsxiConnectionParams): Promise<VmwareInventory>
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
getXapiObject: <T extends XapiXoRecord>(
@@ -517,22 +456,6 @@ export type XoApp = EventEmitter & {
getXoEventEmitterByType(type: string): EventEmitter
hasFeatureAuthorization(featureCode: string): Promise<boolean>
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
listBackupNgDiskPartitions(remoteId: XoBackupRepository['id'], diskId: string): Promise<XoBackupDiskPartition[]>
listBackupNgPartitionFiles(
remoteId: XoBackupRepository['id'],
diskId: string,
partitionId: string | undefined,
path: string
// keys ending in "/" are directories; files carry size (bytes) and mtime (epoch ms).
// Values may be empty ({}) for directories or when served by an older proxy.
): Promise<Record<string, { size?: number; mtime?: number }>>
fetchBackupNgPartitionFiles(
remoteId: XoBackupRepository['id'],
diskId: string,
partitionId: string | undefined,
paths: string[],
format: string
): Promise<Readable>
loadPlugin(id: XoPlugin['id']): Promise<void>
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
@@ -543,25 +466,6 @@ export type XoApp = EventEmitter & {
backupRepositoryIds: XoBackupRepository['id'][],
opts?: { _forceRefresh?: boolean; vmId?: XoVm['id'] }
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]> | null>>
/** imports one VM from VMware, resuming a previous partial import when possible, returns the UUID of the XCP-ng VM */
migrationfromEsxi(
params: EsxiConnectionParams & {
sr: XoSr['id']
network: XoNetwork['id']
/** overrides `sr`, by disk node (e.g. `scsi0:0`) */
disks?: Record<string, XoSr['id']>
/** overrides `network`, by NIC device key (e.g. `4000`) */
nics?: Record<string, XoNetwork['id']>
template: XoVmTemplate['id']
stopSource: boolean
/** VMware moref of the VM */
vm: string
/** dock the guest once the import is complete, off by default. A failed docking does not fail the import */
docking?: boolean
/** start the VM once the import is complete, following the first boot of a docked guest, off by default */
startVm?: boolean
}
): Promise<XoVm['uuid']>
/**
* Serve one disk of a backup archive as an iSCSI LUN and attach it to `host`
* as an SR, read-only unless `cacheSrId` is set. Undone by `unmountBackupArchiveDisk`.
@@ -639,19 +543,6 @@ export type XoApp = EventEmitter & {
): Promise<{ vmUuid: string; success: boolean; merge?: boolean; size?: number; error?: string }[]>
/** Remove a server from the DB (XCP-ng/XenServer) */
unregisterXenServer(id: XoServer['id']): Promise<void>
/**
* Shallow merge of the properties into the existing job, a `null` value removes the property.
*/
updateJob(job: { id: AnyXoJob['id']; [key: string]: unknown }, merge?: boolean): Promise<void>
updateSchedule(params: {
id: XoSchedule['id']
cron?: XoSchedule['cron']
enabled?: XoSchedule['enabled']
jobId?: XoSchedule['jobId']
name?: XoSchedule['name']
timezone?: XoSchedule['timezone']
userId?: XoUser['id']
}): Promise<void>
updateUser(
id: XoUser['id'],
updates: {

View File

@@ -180,15 +180,6 @@ export type XoBackupLog = BaseXoLog & {
export type XoRestoreLog = BaseXoLog & {
message: 'restore'
}
export type XoBackupDiskPartition = {
id: Branded<'xo-backup-disk-partition'>
name?: string
size: number
start: number
/** number for MBR, string for GPT partition */
type: string | number
}
export type XoVmBackupArchive = {
id: Branded<'vm-backup-archive'>
type: 'xo-vm-backup'
@@ -515,68 +506,6 @@ export type XoPlugin =
| XoUsageReportPlugin
| XoWebHooksPlugin
/**
* An ISO selected in a browser and streamed from it, which can be inserted in the CD drive of a VM
* while the browser stays connected
*/
export type XoBrowserMediaSession = {
id: Branded<'browser-media'>
/** File name of the ISO, as selected in the browser */
name: string
/** Size of the ISO, in bytes */
size: number
/** User whose browser serves the ISO */
userId: XoUser['id']
/** Timestamp of the browser connection */
createdAt: number
/** Total of the bytes read from the browser */
bytesRead: number
/** Timestamp of the last read from the browser */
lastReadAt?: number
} & (
| {
/** Not inserted in any VM */
status: 'connected'
vmId?: undefined
vdiId?: undefined
insertedAt?: undefined
/** Why the last insertion failed */
error?: string
}
| {
/** Being inserted in a VM */
status: 'inserting'
/** VM the ISO is being inserted in */
vmId: XoVm['id']
vdiId?: undefined
insertedAt?: undefined
error?: undefined
}
| {
/** Inserted in the CD drive of a VM */
status: 'inserted'
/** VM the ISO is inserted in */
vmId: XoVm['id']
/** VDI inserted in the CD drive of the VM */
vdiId: XoVdi['id']
/** Timestamp of the end of the insertion */
insertedAt: number
error?: undefined
}
| {
/** Being ejected, or a failed insertion being undone */
status: 'ejecting'
/** VM the ISO is ejected from */
vmId: XoVm['id']
/** VDI inserted in the CD drive of the VM, unless the insertion failed before */
vdiId?: XoVdi['id']
/** Timestamp of the end of the insertion, unless it failed */
insertedAt?: number
/** Why the insertion failed */
error?: string
}
)
export type XoProxy = {
id: Branded<'proxy'>
url: string
@@ -601,22 +530,15 @@ type XoBackupJobGeneralSettings = {
[key: string]: unknown
}
/**
* Keeps the last backup of each of the last `retention` days/weeks/months/years, in addition to `exportRetention`.
*
* Requires the `timezone` setting.
*/
export type XoBackupJobLongTermRetention = {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
cbtDestroySnapshotData?: boolean
concurrency?: number
longTermRetention?: XoBackupJobLongTermRetention
longTermRetention?: {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
checkpointSnapshot?: boolean
offlineSnapshot?: boolean
maxExportRate?: number
@@ -630,8 +552,8 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
}
export type XoVmBackupJobScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags'] | string
fullInterval?: number | string
healthCheckVmsWithTags?: XoVm['tags']
fullInterval?: number
copyRetention?: number
snapshotRetention?: number
cbtDestroySnapshotData?: boolean
@@ -642,9 +564,9 @@ export type XoVmBackupJob = BaseXoJob & {
compression?: 'native' | 'zstd' | ''
proxy?: XoProxy['id']
mode: 'full' | 'delta'
remotes?: CMType.IdOr<XoBackupRepository['id']> | string
remotes?: CMType.IdOr<XoBackupRepository['id']>
vms: CMType.IdOr<XoVm['id']> | Record<string, unknown>
srs?: CMType.IdOr<XoSr['id']> | string
srs?: CMType.IdOr<XoSr['id']>
type: 'backup'
settings: {
'': XoVmBackupJobGeneralSettings
@@ -673,17 +595,16 @@ export type XoMetadataBackupJob = BaseXoJob & {
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
concurrency?: number
longTermRetention?: XoBackupJobLongTermRetention
nRetriesVmBackupFailures?: number
mergeBackupsSynchronously?: boolean
timeout?: number
timezone?: string
maxExportRate?: number
backupReportTpl?: 'compactMjml'
reportWhen: 'failure'
}
export type XoMirrorBackupScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags'] | string
vmFilter?: string
healthCheckVmsWithTags?: XoVm['tags']
healthCheckSr?: XoSr['id']
[key: string]: unknown
}
@@ -825,7 +746,6 @@ export type XoTask = {
| 'backup-job'
| 'backup-log'
| 'backup-repository'
| 'browser-media'
| 'group'
| 'plugin'
| 'proxy'
@@ -1042,7 +962,6 @@ export type NonXapiXoRecord =
| AnyXoJob
| AnyXoLog
| XoAuthenticationToken
| XoBrowserMediaSession
| XoGroup
| XoPlugin
| XoProxy

View File

@@ -1,5 +1,3 @@
export default {
export: true, // can export the content ( full or file by file )
read: true, // can read the metadata of an archive
mount: true, // can mount the archive for file level restore
read: true,
}

View File

@@ -1,11 +1,6 @@
export default {
read: true,
update: {
dns: true,
gateway: true,
ip: true,
management: true,
mode: true,
netmask: true,
},
}

View File

@@ -1,583 +0,0 @@
VMs Auto-pilot docking prepares a VM migrated from VMware to boot on XCP-ng, before its guest OS ever
boots there. It is made of two parts:
- **a live ISO**, built from this folder: booted in the migrated VM, it inspects the guest disks and
applies offline what the guest needs on XCP-ng (Xen PV drivers in the initramfs, VMware tools
disabled, XCP-ng guest tools installed, network configuration carried over…), then powers the VM
off. It runs inside the target VM, so it sees the real XCP-ng virtual hardware;
- **the orchestration in xo-server** (`packages/xo-server/src/xo-mixins/vmware/`), enabled by the
`docking` option of the import from VMware, and `startVm` to follow the first boot.
XO and the ISO talk through `VM.xenstore_data`: XO writes the inputs before booting the ISO, the ISO
writes its state and a compact report, see [xenstore contract](#xenstore-contract).
### Life cycle of a migrated VM
Each step is a state of the VM, each transition the condition XO waits for. The VM boots 2 to 3 times
on the ISO, then once on its disk for Linux and twice for Windows (plus one reboot to enable Secure
Boot again, when the source had it).
```mermaid
flowchart TD
S0[["0 · import complete<br/>VM halted, start blocked<br/>snapshot: complete import from V2V"]]
S1["1 · boot on the ISO, diagnose<br/>XO: ISO in the CD drive, boot order dc,<br/>guest tools ISO as a read-only disk, Secure Boot off<br/>ISO: reads the disks, writes the report, powers off"]
S2["2 · XO moves the OS disk to userdevice 0"]
S3["3 · boot on the ISO, dock<br/>XO: nics (Linux), windows-nics and windows-reboot (Windows)<br/>ISO: applies the actions, writes report and log in the guest, powers off"]
S4["4 · XO: ISO ejected, tools disk detached, boot order restored,<br/>xenstore keys cleared, timeoffset (Windows),<br/>start unblocked"]
S5["5 · boot on the disk"]
L6["6L · Linux boots on the PV drivers<br/>SELinux relabel: one more reboot, by the guest<br/>kdump rebuilds its initrd"]
W6["6W · first boot, phase drivers<br/>installs the PV drivers and the agent<br/>out/firstboot = drivers:reboot, shuts down"]
W7["7W · XO starts the VM, phase finish<br/>static addresses from network.ini<br/>out/firstboot = done, first-boot service deleted"]
WF["XO stops the VM and boots the ISO in diagnose<br/>to read the state of the first boot offline"]
S8["8 · XO waits for the guest agent, 15 min at most"]
S9["9 · Secure Boot on, clean reboot<br/>without agent after it: Secure Boot off, VM started again"]
E0(["VM running on XCP-ng"])
E1(["VM halted and docked:<br/>Windows reboots by itself between its phases"])
E2(["VM halted: FIRST_BOOT_FAILED<br/>C:\xo-docking\firstboot.log"])
E3(["VM running on emulated devices, without agent"])
EF(["VM halted, not docked: reason in the task<br/>the snapshot of the import is the rollback"])
S0 -->|"docking option"| S1
S1 -->|"boot_disk_not_first, once"| S2
S2 --> S1
S1 -->|"other blocking finding, or no result in 30 min"| EF
S1 -->|"nothing blocking"| S3
S3 -->|"result = error"| EF
S3 -->|"result = ok or warning"| S4
S4 -->|"startVm off"| E1
S4 -->|"startVm on"| S5
S5 -->|"Linux"| L6
S5 -->|"Windows, build 14393 and later"| W6
S5 -->|"Windows before build 14393"| E3
L6 --> S8
W6 -->|"VM halted"| W7
W7 -->|"out/firstboot = done"| S8
W6 & W7 -->|"failed, halted again, or 45 min"| WF
WF --> E2
S8 -->|"agent up, Secure Boot on the source"| S9
S8 -->|"no Secure Boot on the source, or no agent in 15 min"| E0
S9 --> E0
classDef ok fill:#d4edda,stroke:#2e7d32,color:#1b3d1f
classDef halted fill:#fff3cd,stroke:#b58105,color:#4d3800
classDef failed fill:#f8d7da,stroke:#c62828,color:#4a1015
class E0,E3 ok
class E1 halted
class E2,EF failed
```
Boot by boot:
| Boot | On | Started by | What runs | Ends with |
| ---- | ---- | --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- |
| 1 | ISO | XO | `diagnose`: read-only inspection, planned actions, blocking findings, `rtc_timeoffset` and network of Windows | power off by the ISO |
| 1b | ISO | XO | `diagnose` again, only after XO moved the OS disk first (`boot_disk_not_first`) | power off by the ISO |
| 2 | ISO | XO | `dock`: the planned actions, in order; full report and log written in the guest | power off by the ISO |
| 3 | disk | XO (`startVm`) or the administrator | Linux: the guest itself, SELinux relabel if scheduled. Windows: phase `drivers` of the first boot, PV drivers and agent | Linux: running. Windows: shut down (`windows-reboot=xo`) or reboot (`guest`) |
| 4 | disk | XO (`windows-reboot=xo`), the guest otherwise | Windows only: phase `finish`, static addresses reapplied, first-boot service deleted | running, `out/firstboot = done` |
| 5 | disk | XO (`VM.clean_reboot`) | only when the source had Secure Boot and the pool has its UEFI certificates | running, agent reporting |
A failed docking never fails the import: the data is there, and the snapshot taken at the end of the
import is the rollback. The task of the import carries the detail of each run and of the first boot.
### Limits
The VM is imported but not docked, and left halted, when:
- the import is partial: the source was running and `Stop source` was not set. The task says so, the
VM can be docked once a later import completes it;
- no docking ISO is deployed in an ISO SR of the pool (`DOCKING_ISO_NOT_FOUND`);
- its system volume is encrypted (LUKS, BitLocker);
- it is a Windows before 7 SP1 / 2008 R2 SP1, on dynamic disks, hibernated, with Fast Startup, or not
cleanly shut down;
- no Linux or Windows is found on its disks (BSD...).
The task of the import gives the reason.
It is docked, with less than a full migration, when:
- it is a Windows before build 14393: no PV drivers and no management agent;
- it is a Windows with several NICs whose source did not report its addresses (not running, or without
VMware Tools): its static addresses are not carried over, the adapters use DHCP;
- the source had Secure Boot: it is enabled again only when XO follows the first boot (`startVm`);
- the guest is in [partially supported](#partially-supported).
Only imports from VMware use it.
### Supported guests
"Validated" means: installed on ESXi 8 with the VMware defaults (PVSCSI or LSI SAS, vmxnet3 or
e1000e, open-vm-tools / VMware Tools), imported with XO, docked by the ISO, then booted on XCP-ng with
no manual step. The guest runs on PV disk and NIC, and XO gets the OS, the agent version, the memory
and the IP addresses.
#### Fully supported
| Family | Validated | Initramfs | Guest agent | Network configuration |
| ----------- | -------------------------------------------------------------------------- | ------------------------- | -------------------------------- | ------------------------------------- |
| Debian | 10, 11, 12 (BIOS), 13 (BIOS, UEFI) | initramfs-tools | `.deb` | ifupdown |
| Ubuntu | 22.04 (UEFI), 24.04 (BIOS), 26.04 desktop (UEFI) | initramfs-tools | `.deb` | netplan, NetworkManager |
| RHEL family | CentOS 7.3, Rocky 8.10, 9.8, 10.2, Oracle Linux 9.8 (UEK and RHCK kernels) | dracut | `.rpm` | ifcfg, NetworkManager |
| SUSE | SLES 16.0, openSUSE Leap 15.6 (BIOS, UEFI), SLES 15 SP7 (UEFI) | dracut | `.rpm` | NetworkManager, wicked |
| Arch | rolling (BIOS) | mkinitcpio (experimental) | generic `.tgz` | systemd-networkd |
| Windows | Server 2016, 2019, 2022, 2025 | n/a | PV drivers 9.x, management agent | captured, reapplied by the first boot |
Configurations validated across these guests: BIOS and UEFI, Secure Boot off on the source, SELinux
enforcing (relabel), btrfs with snapper, LVM over three disks with the root LV spanning two, `/boot`
and `/usr` each on their own disk, fstab by UUID, by `dm-uuid` and by YaST device ID, two NICs with
static addresses on both (Linux, and Windows Server 2022 with `windows-nics`: each address on its VIF,
DNS queries seen on the first one), Windows static address with its DNS reapplied by the first boot
(Server 2019 and 2022), DHCP (Ubuntu 22.04 gets the same address on XCP-ng as on VMware: same MAC,
same lease), kdump enabled.
RHEL itself is not tested (no subscription on the test bench): Rocky Linux and Oracle Linux (RHCK
kernel) share its kernel, dracut, rpm and NetworkManager; the RHEL subscription is left untouched.
#### Partially supported
| Guest | What works | What is missing |
| ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Windows before build 14393: 7 SP1, 8.1 and Server 2012 R2 validated, 2008 R2 expected the same | Offline changes; the VM boots on the emulated IDE disk and the emulated NIC (inbox driver), with the VMware MAC: DHCP and MAC reservations keep working, a static IP is reapplied by the first boot | The XCP-ng 9.x PV drivers do not support these versions (`pv_drivers_unsupported`): by default the first boot installs no package. No PV drivers, no management agent, so no `out/firstboot` in xenstore and XO does not follow the first boot: its state is only readable offline (`diagnose`, `os.firstboot`) |
| Transactional systems with a read-only root (SL Micro 6.2) | Nothing is written offline. The VM boots on PV devices when its initramfs already has the Xen drivers, which the ISO checks (`initramfs_xen_drivers_present`, true for the generic SL Micro image) | The guest agent and the other offline changes are listed in `readonly_root`, to do after the first boot with `transactional-update`. An initramfs without the Xen drivers is blocking (`readonly_root_initramfs`): rebuild it on the source first |
| Linux with neither systemd nor a package manager supported by the guest tools | Initramfs, fstab, VMware tools, network | No guest agent (`guest_agent_unsupported`): install it after the first boot |
| Linux without systemd-udevd | Initramfs, fstab, VMware tools, agent | Interface names are not pinned (`linux_interface_names`) |
| Linux with a kernel without `xen-blkfront` | Boots on the emulated IDE disk (`no_xen_blkfront`) | No PV disk |
#### Not supported
- Windows before 7 SP1 / 2008 R2 SP1 (`unsupported_windows_version`).
- Encrypted system volume, LUKS root or BitLocker on the Windows volume: the ISO has no key, so it
reports the volume (`luks_volume`, `bitlocker_volume`) and finds no system to change
(`no_os_found`, blocking). Decrypt it on the source before the migration. An encrypted data volume,
next to a system in clear, is reported and left untouched: the guest is docked, the volume itself is
not tested.
- Windows dynamic disks, hibernated Windows or Fast Startup: blocking.
- Windows volume not cleanly shut down (`ntfs_unclean`, blocking): the source was powered off, or
imported while running and then powered off. ntfs-3g only mounts such a volume read-only, so nothing
could be written. The source must be shut down from the guest (VMware Tools, which `Stop source`
does), not powered off.
- Anything else (BSD, ...): `no_os_found`. When MD RAID or LVM could not be activated, the report
also has a `storage_activation_failed` warning per failed command, with its error: the system may
be on them.
### What the ISO does
Detection never modifies anything: it records findings and plans actions, each with a stable `code`.
`dock` applies them in order. Non-critical actions only produce a warning when they fail.
#### All Linux guests
| Action | Code | What it does |
| ------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Initramfs | `initramfs_xen_drivers` | Rebuilds it with the guest's own tool in a chroot, with `xen-blkfront`, `xen-netfront` and the emulated IDE drivers. Critical: a PVHVM kernel unplugs the emulated disks at boot whenever it has blkfront support, so without the module in the initramfs the root disk disappears |
| fstab | `fstab_stable_names` | Rewrites `/dev/sdX`, and hardware `by-id` / `by-path` names of the root disk, to `UUID=`. In place (inode and SELinux label kept), backup in `fstab.xo-docking.bak`. Content-derived names (`dm-uuid-`, `lvm-pv-uuid-`, `md-uuid-`) are stable and left alone |
| VMware tools | `disable_vmware_tools` | Disables the open-vm-tools / vmware-tools units (`vmtoolsd`, `vgauth`, `vgauthd`). Masked units are left alone |
| Network | `linux_pin_interface_names` | Interface names change on XCP-ng (`ens192` → `enX0`). For each old name found in the network configuration, writes `/etc/systemd/network/10-xo-docking-<name>.link`, which gives the name back to the NIC with the same MAC, before the initramfs is rebuilt. The guest configuration (ifupdown, ifcfg, NetworkManager, netplan, networkd) then applies unchanged. Automatic with one NIC; with several, XO passes `nics`. Kernel names (`eth0`) do not change. The network configuration itself is never modified: static and DHCP interfaces stay as they are, and a DHCP reservation by MAC keeps working since the VIFs keep the VMware MACs |
| Guest agent | `install_guest_agent` | When neither `xe-guest-utilities` nor `xen-guest-agent` is there, installs `xe-guest-utilities` from the [guest tools disk](#guest-tools-disk) and enables `xe-linux-distribution` (not started) |
| kdump | `kdump_initrd_rebuild` | Deletes the kdump initrd built on VMware without `xen-blkfront`; kdump rebuilds it on XCP-ng at the next boot. Without this, the capture kernel does not find the disk |
| SELinux | `selinux_relabel` | When SELinux is enabled and anything was changed, schedules a relabel (`/.autorelabel`), which may cost one extra reboot |
#### Per family
- **Debian, Ubuntu**: initramfs-tools (`update-initramfs -u -k all`, modules appended to
`/etc/initramfs-tools/modules`). Agent: the `.deb`, installed with `dpkg -i` under a temporary
`policy-rc.d` that forbids starting services: in a chroot, `invoke-rc.d` would otherwise start the
daemon on the live system.
- **RHEL family** (RHEL, CentOS, Rocky, Alma...): dracut (`--add-drivers`, per kernel). Agent: the
`.rpm` (`rpm -U`; the scriptlets enable the unit and `systemctl` starts nothing in a chroot), or the
`legacy` `.rpm` with `chkconfig` when there is no systemd. kdump: `/boot/initramfs-*kdump.img`,
rebuilt by `kdumpctl`.
- **SUSE** (SLES, openSUSE): dracut. Root on btrfs with snapper: the default subvolume is mounted,
`subvol=` options of fstab are honoured. fstab by device ID (YaST "mount by device ID", older SLES)
is rewritten when it names the root disk: its entry tells which VMware name is that disk, and the
partitions follow. kdump: `/var/lib/kdump/initrd`, rebuilt by `mkdumprd`. Agent: the `.rpm`.
- **Arch and other systemd guests without dpkg or rpm**: mkinitcpio (drop-in
`/etc/mkinitcpio.conf.d/90-xo-docking.conf`, `mkinitcpio -P`), reported as experimental. Agent: the
generic `.tgz`, extracted by the ISO following the guest's own links (`/usr/sbin` → `usr/bin`), with
the systemd unit of the guest tools ISO.
- **Transactional systems** (SL Micro, root on a read-only btrfs snapshot, or `/` mounted `ro` in
fstab with `transactional-update` present): the root is never written offline. Instead of
rebuilding the initramfs, the ISO lists it (`lsinitramfs`) and checks that `xen-blkfront` and
`xen-netfront` are in it for every kernel.
#### Windows (7 SP1 / 2008 R2 SP1 and later)
Offline, from the ISO, in every control set so that "Last Known Good Configuration" boots too. The
SYSTEM hive is backed up as `SYSTEM.xo-docking.bak` before being written (`commit_registry`).
| Action | Code | What it does |
| --------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Boot drivers | `enable_ide_boot` | `atapi`, `intelide`, `pciide` start at boot, `StartOverride` removed, so Windows boots from the emulated IDE disk |
| VMware services | `disable_vmware_services` | Disables them (Start=4), found by name and by rule (image path under `\VMware\VMware Tools\`, or a `vm*` kernel driver described as VMware). Conservative: other VMware/Broadcom products (Carbon Black...) are left alone |
| Network | `clear_old_static_ip` | Captures the TCP/IP configuration of the old PCI adapters in the report (`network`), then switches the ones with a static address to DHCP; otherwise Windows refuses the same address on the new adapter. Adapters already in DHCP are left untouched. The new PV adapter starts in DHCP with the VMware MAC, so a DHCP reservation by MAC applies to it as before. The static configuration (addresses, masks, gateways, DNS) is written to `\xo-docking\network.ini` for the first boot, keyed by the MAC of the VIF: automatic with a single static adapter on a single NIC, from `windows-nics` otherwise (`windows_static_ip_unmapped` when an adapter has no MAC) |
| First boot | `install_firstboot` | Installs the temporary service `XoDockingFirstboot` and `\xo-docking\` (`firstboot.ps1`, `config.ini`), and copies the packages matching `windows-msi-patterns` from the guest tools disk into `\xo-docking\packages\` |
Also reported: `os.rtc_timeoffset` (seconds, which XO sets as `platform:timeoffset`: Windows keeps the
RTC in local time and XCP-ng starts it at UTC), the state of a previous first boot (`os.firstboot`,
`windows_firstboot_incomplete` with the end of its log), and `pv_drivers_unsupported` below build 14393. Below that build, the default `windows-msi-patterns` is empty: the XCP-ng 9.x MSI would fail and
stop the first boot in its `drivers` phase, before `finish` reapplies the static IP. Patterns given
explicitly by XO are always used.
##### Windows first boot
`firstboot.ps1` runs at each boot as LocalSystem until its phases are done, then deletes its service.
It is compatible with PowerShell 2.0. Before any phase, it restricts `C:\xo-docking` to SYSTEM and the
administrators (written through ntfs-3g, it is writable by every user), and removes the `Run` entries
that start a program from `\VMware\VMware Tools\` ("VMware User Process"), which crashes without its
services.
| Phase | `standard` | `full` | What it does | Ends with |
| --------- | ---------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------- |
| `drivers` | yes | yes | installs the packages matching `windows-msi-patterns` from `\xo-docking\packages\`, in order, then tries the VMware Tools MSI uninstall | reboot |
| `cleanup` | | yes | removes VMware driver packages from the driver store (`pnputil -f -d`, provider read from the INF), the VMware Tools directories, and the VMware-signed `vm*.dll` / `vm*.sys` / `pvscsi.sys` from System32 | reboot |
| `finish` | yes | yes | applies `network.ini` to the adapters with these MACs (WMI `EnableStatic`, `SetGateways`, `SetDNSServerSearchOrder`). `full` only: removes the ghost VMware PCI devices (vendor 15AD) with `pnputil /remove-device`, on Windows 10 2004 / Server 2022 and later. Then reports `done` | service deleted |
With `windows-reboot=xo`, each reboot of the table is a shut down instead (`shutdown /s`): XO, which
follows the first boot, starts the VM again. Before the PV drivers are loaded the guest cannot tell it
anything, so a stopped VM is the signal. Once they are loaded, the script also writes
`<phase>:reboot` before shutting down. The default, `guest`, is for a VM started by hand: it must never
be left stopped in the middle of its first boot. XO sets `xo` when it follows the first boot
(`startVm`) and the PV drivers support the guest.
A failing phase is retried at the next boot, up to 3 attempts, then the script gives up
(`<phase>:failed`) and deletes its service. The log is in `\xo-docking\firstboot.log`, next to the MSI
installation logs.
`standard` is the default and non-destructive: the VMware components stay on disk, disabled. `full`
follows [vmwremover](https://github.com/virtio-win/vmwremover), except that vsock is kept (removing it
means editing the Winsock catalog) and devcon.exe is not used (not redistributable): ghost devices stay
where pnputil cannot remove them, which is harmless since the old static addresses were cleared.
#### UEFI (all guests)
The NVRAM on XCP-ng is new, so the boot entries created on VMware are lost. The ISO does not write
NVRAM: it makes sure `\EFI\BOOT\BOOTX64.EFI` exists on the ESP (`efi_fallback_loader`). For Linux it
copies shim, GRUB and grub.cfg from the vendor directory, like `grub-install --force-extra-removable`;
for Windows, `bootmgfw.efi`.
#### Detected but not changed
- **Interface names that could not be pinned**: several NICs without `nics`, or no systemd-udevd
(`linux_interface_names`).
- **Unstable device names left**: hardware by-id/by-path names of other disks than the root one
(`fstab_unstable_names`), `resume=` or `root=` with such a name in `/etc/default/grub`
(`grub_unstable_resume`, `grub_unstable_root`): the initramfs waits for that device at every boot.
- **Firmware mismatches** (Windows on GPT with BIOS, UEFI without ESP).
- **OS not on the first disk** (`boot_disk_not_first`, blocking: nothing is modified). XCP-ng boots
the first disk, so such a VM does not boot. XO moves the VBD of `data.os_disk` to `userdevice` 0 and
runs the ISO again. Validated with Debian 13 (data disk first, OS on the second disk).
- **Encryption, dynamic disks, hibernation**: see [not supported](#not-supported).
#### Left to the administrator
- **open-vm-tools / VMware Tools** stay installed, disabled (`standard` cleanup on Windows). Remove
them when convenient.
- **Secure Boot**: off during the ISO runs (the ISO is not signed). With `startVm`, XO turns it back on
after the first boot if the source had it and the pool certificates are installed, and turns it off
again if the VM does not boot with it: validated on Ubuntu 22.04, SLES 16.0 and Windows Server 2022.
Without `startVm`, enable it again once the VM has booted.
- **SCC / RHN registrations**: not tested.
### XO side
The orchestration is the [life cycle](#life-cycle-of-a-migrated-vm) above. Details that matter:
- **The ISO** is the most recent `vates-autopilot-docking-*.iso` (by name, which starts with the build date and time) of an
ISO SR of the pool, inserted in the CD drive of the VM (created when it has none) and ejected after
the runs. `DOCKING_ISO_NOT_FOUND` when there is none.
- **`platform:secureboot=false` is mandatory for every UEFI VM** on each boot on the ISO, even when XO
shows `secureBoot: false`: an absent key means `auto` on XCP-ng 8.3. Symptom: the VM lands in the
UEFI Shell (`Access Denied` on `\EFI\BOOT\BOOTX64.EFI`) and nothing appears in xenstore. A signed ISO
would remove this.
- **`nics`** is built from the networks of the VMX, `ens<ethernetN.pciSlotNumber>=<MAC>` for each NIC:
the predictable name the guest gave it on VMware (`pciSlotNumber` 192 → `ens192`). With it, a
migration with several NICs needs no manual step.
- **`windows-nics`** matches the addresses the VMware tools reported while the source was running (MAC
→ addresses) with the `network` of the diagnose report (GUID → addresses). A Windows source with
several NICs which reported nothing (not running, no VMware tools) keeps DHCP on all its adapters,
with a warning.
- **Running the ISO again** (retry after a failure or a timeout) is safe and validated on Rocky 9 and
Windows Server 2022: on Linux the agent is not reinstalled and fstab and the pinned names stay as
they are (the initramfs is rebuilt and a relabel scheduled again); on Windows, a first boot that
already ended in `done` with the PV drivers installed is not scheduled again
(`windows_firstboot_already_done`). But the old adapters were switched to DHCP by the first run, so
the second report has no static configuration in `network`: the `network` of the first report is the
one to keep.
- **Stale keys**: `VM.xenstore_data` persists after the halt and `vm.set` merges keys, so before each
boot on the ISO XO deletes every input it does not set and the whole `out/` subtree, and deletes all
of them after the last run. Otherwise it may read a `done` from a previous run. The ISO also deletes
`out/` when it starts, but too late to protect XO.
- **Timeouts**: 30 minutes for a run of the ISO, 45 minutes for the Windows first boot, 15 minutes for
the guest agent to report after the first boot. XO does not wait for an agent the docking could not
install (`guest_agent_missing`, `guest_agent_unsupported`, failed `install_guest_agent`): it warns
right away.
#### Not done by XO
- The fallback for the adapters left in `windows_static_ip_unmapped` (`VIF.configure_ipv4` /
`VIF.configure_ipv6` once the agent is up, validated by hand on Windows Server 2019 with agent
9.2.385: address and gateway applied within seconds, no DNS). They stay in DHCP, with a warning.
- `windows-cleanup=full`: XO always uses `standard`.
- Windows patches (`.msp`): `firstboot.ps1` runs `msiexec /i` on every package. Not needed with the
current tools ISO (its `XenTools-fix-9.2.351.msp` targets 9.2.351, the MSI installs 9.2.385). If a
future one needs it: `/p` by extension, after its MSI, 1642 (not applicable) as skipped.
#### Guest tools disk
The ISO takes the XCP-ng guest tools from the guest tools ISO every pool already has, so the docking
ISO stays generic and the tools match the host.
- **VDI**: the guest tools ISO of the pool (`VDI.is_tools_iso = true`, in the `XCP-ng Tools` SR); XAPI
accepts a VDI of an ISO SR in a disk VBD.
- **VBD**: `type=Disk`, `mode=RO`, `userdevice=25` (`xvdz`) when free, `bootable=false`. Not a CD:
xen-blkfront ignores CD drives on HVM guests, so the live system would not see it.
- **Recognition**: a whole disk with an `iso9660` or `udf` filesystem and a `Linux/` or `Windows/`
directory. It is never taken as a guest disk.
- **Optional**: without it the report says `"tools": null`, Linux gets `guest_agent_missing` and
Windows `windows_tools_disk_missing`; the Windows first boot then has no package to install, and
gives up after its attempts.
#### xenstore contract
Paths are relative to the domain (`/local/domain/<domid>/`).
Inputs, set by XO through `VM.xenstore_data` (keys must start with `vm-data/`):
| Key | Values | Default |
| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- |
| `vm-data/xo-docking/mode` | `diagnose`, `dock` | `diagnose` |
| `vm-data/xo-docking/windows-msi-patterns` | `;`-separated file patterns of the packages to install, in order (e.g. `xen-*.msi;managementagent-*.msi`) | `*.msi`, none below build 14393 |
| `vm-data/xo-docking/windows-cleanup` | `standard` or `full`, see [Windows first boot](#windows-first-boot) | `standard` |
| `vm-data/xo-docking/windows-reboot` | `guest` or `xo`: who restarts the VM between the phases of the first boot, see [Windows first boot](#windows-first-boot) | `guest` |
| `vm-data/xo-docking/nics` | Linux, `;`-separated `<old name>=<MAC>` (e.g. `ens192=00:0c:29:d4:bf:19;ens224=00:0c:29:d4:bf:23`). Only needed with several NICs | unset |
| `vm-data/xo-docking/windows-nics` | Windows, `;`-separated `<MAC>=<GUID>`: the VIF that gets the static configuration of the old adapter with this `guid` in `network` (e.g. `00:0c:29:94:80:0d={44FED271-EE81-4363-8CF0-893542F0734F}`). Only needed with several NICs | unset |
| `vm-data/xo-docking/keep-running` | `1` to stay up after the run (debugging) | unset |
Outputs, written by the ISO under `vm-data/` too: XAPI syncs the guest's `vm-data/` subtree back into
`VM.xenstore_data`, but not `data/`.
| Key | Content |
| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `vm-data/xo-docking/out/version` | ISO version (`YYYYMMDD-<sha>` or the release tag) |
| `vm-data/xo-docking/out/mode` | mode actually used |
| `vm-data/xo-docking/out/state` | `running`, then `done` (written last: the other keys are final) |
| `vm-data/xo-docking/out/result` | `ok`, `warning` or `error` |
| `vm-data/xo-docking/out/report` | compact JSON report, written raw (`xenstore-write -R`) |
| `vm-data/xo-docking/out/firstboot` | Windows, written by the first-boot script once the PV drivers are loaded: `<phase>:running`, `<phase>:reboot` (`windows-reboot=xo`, the VM shuts down), `<phase>:error`, `<phase>:failed` or `done`. A later `diagnose` run reads it offline and writes it too (`<phase>:pending` if it never started) |
oxenstored refuses guest values over 2048 bytes (`quota-maxsize`), so the compact report is limited
to 1900 bytes. What XO never reads goes first: `os` is reduced to `kind`, `name`, `rtc_timeoffset` and
`firstboot`, `network` to the static adapters with their `guid`, `dhcp` and `addresses`. Then the
details of the findings, the info-level findings and the messages. Last, findings and actions are
dropped one by one, those XO acts on last: errors, failed actions, and the codes in `XO_CODES` of
`report.py` (`boot_disk_not_first`, `guest_agent_missing`, `guest_agent_unsupported`,
`install_firstboot`, `install_guest_agent`, `pv_drivers_unsupported`, `windows_firstboot_incomplete`,
`windows_static_ip_unmapped`), info ones included. In dock mode the full report and the log are also
written into the guest: `/var/log/xo-docking/` on Linux, `\xo-docking\` on Windows.
```json
{
"version": "20261012T084512Z-1a2b3c4d",
"mode": "diagnose",
"result": "warning",
"firmware": "uefi",
"disks": ["xvda", "xvdb"],
"tools": "xvdz",
"os": {
"kind": "linux",
"name": "Debian GNU/Linux 12 (bookworm)",
"id": "debian",
"version": "12",
"root": "/dev/xvda2",
"kernels": ["6.1.0-18-amd64"],
"initramfs": "initramfs-tools",
"guest_agent": "xe-guest-utilities_7.30.0-19_amd64.deb"
},
"network": [],
"findings": [
{ "level": "warning", "code": "linux_interface_names", "message": "...", "data": { "interfaces": ["ens192"] } }
],
"actions": [{ "code": "initramfs_xen_drivers", "status": "planned", "description": "..." }]
}
```
For Windows, `network` lists the TCP/IP configuration of the old adapters (`adapter`, `vmware`,
`dhcp`, `addresses`, `masks`, `gateways`, `dns`).
### Troubleshooting
Look at the task of the import, then at the files in the guest.
xenstore only helps while the ISO or the Windows first boot runs: XO deletes its keys after its runs.
#### The task of the import
Each step is a subtask, with its own status: `attaching the docking ISO`, a run of the ISO,
`releasing the docking ISO`, then for the first boot `starting the VM`, on Windows
`<phase> phase of the Windows first boot` and `restarting the VM after the <phase> phase of its first boot`,
and `waiting for the guest agent`. A phase of the Windows first boot fails when it does not report in
time, gives up, or stops the VM once more.
Each run of the ISO is a subtask, `docking ISO: diagnose` or `docking ISO: dock`, whose log carries the
version of the ISO, the result, the guest OS, then one line for each action (`action <code>: <status>`)
and each finding (`finding <level> <code>: <message>`). In a `dock`
run, a failed action and a finding above `info` are warnings of the task; a `diagnose` run is only a
plan, none of its lines is a warning.
| Line of a run of the ISO | Meaning |
| --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
| `docking ISO <version>: <mode> ok`, `warning` or `error` | result of the run |
| `action <code>: <status> (<error>)` | `planned` (diagnose only), `applied`, `failed`, or `skipped` after a failed critical action. The codes are in [what the ISO does](#what-the-iso-does) |
| `finding <level> <code>: <message>` | an `error` is blocking |
| `details were dropped from the report to fit in xenstore` | the full report is in the guest |
A blocking finding stops the docking after `diagnose`: the guest is as imported. When Windows stops
between the phases of its first boot for XO to start it again (`windows-reboot=xo`) and XO no longer
follows it, the warning of the task says to start the VM again until `firstboot.log` says it is done.
A failure does
not fail the import: it is a warning of the task, `the guest could not be docked: <message>` or
`the first boot on XCP-ng failed: <message>`, with one of these codes in the log of xo-server:
| Code | Meaning | What to do |
| --------------------------- | ------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
| `DOCKING_ISO_NOT_FOUND` | no `vates-autopilot-docking-*.iso` in an ISO SR of the pool. Checked before any transfer: this one fails the import | deploy the ISO, then import again |
| `DOCKING_ISO_TIMEOUT` | no result within 30 minutes: most often the firmware did not boot the ISO | [run the ISO by hand](#running-the-iso-by-hand) and watch the console: the UEFI Shell with `Access Denied` means Secure Boot is on |
| `DOCKING_ISO_NO_RESULT` | the VM halted without the ISO writing `done`, or without a report | run the ISO by hand with `keep-running`, read `/run/xo-docking/xo-docking.log` |
| `DOCKING_OS_DISK_NOT_FOUND` | the ISO found the OS on a disk the VM does not have | check the disks of the VM |
| `DOCKING_BLOCKED` | a blocking finding in `diagnose`: nothing was changed | the finding is in the task of the `diagnose` run, see [limits](#limits) |
| `DOCKING_FAILED` | the `dock` run failed: the guest may be partly changed | the files in the guest; revert to the snapshot of the import to start again |
| `FIRST_BOOT_FAILED` | the Windows first boot did not end: XO stopped the VM and read its state offline | `C:\xo-docking\firstboot.log` |
The snapshot of the import is `after complete import from V2V`, or `complete import from V2V` when the
source was stopped by the import (`Stop source`).
Warnings without failure: no guest tools ISO in the pool (no guest agent), static addresses not
carried over (`windows_static_ip_unmapped`), guest agent not reporting within 15 minutes of the first
boot, no guest agent installed by the docking, Secure Boot left off.
#### xenstore
XO deletes every `vm-data/xo-docking/` key before each run of the ISO and after the last one. After the
docking, only `vm-data/xo-docking/out/firstboot` can be there: the Windows first boot writes it once the
PV drivers are loaded, so it holds the last state it could report. XO deletes it too once the first
boot is done: it is only left when XO did not follow the
first boot to its end. From the pool master:
```sh
xe vm-param-get uuid=<VM UUID> param-name=xenstore-data param-key=vm-data/xo-docking/out/firstboot
```
During a run of the ISO, all the [outputs](#xenstore-contract) are there:
`xe vm-param-get uuid=<VM UUID> param-name=xenstore-data`.
#### The files in the guest
The report in xenstore is short on purpose (codes always, details when they fit). The details stay in
the VM: `dock` writes the full report and the full log of the ISO into the guest, next to what the
first boot produces. A `diagnose` run writes nothing.
| Guest | Files |
| ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Linux | `/var/log/xo-docking/report.json`, `/var/log/xo-docking/xo-docking.log`; backups next to the changed files (`/etc/fstab.xo-docking.bak`, `/etc/initramfs-tools/modules.xo-docking.bak`) |
| Windows | `C:\xo-docking\report.json`, `xo-docking.log` (ISO), `firstboot.log` (first boot, each phase and the static IP configuration), `install-*.log` (MSI), `network.ini`; `SYSTEM.xo-docking.bak` next to the SYSTEM hive |
When the guest does not boot, reach them by [running the ISO by hand](#running-the-iso-by-hand) with
`keep-running`, or by attaching the disk to another VM.
#### Running the ISO by hand
To watch a run on the console, or to reach the disks of a guest which does not boot. On the pool
master, with the VM halted:
```sh
vm=<VM UUID>
# the CD drive XO created; without one: xe vm-cd-add uuid=$vm cd-name=<ISO name> device=3
xe vm-cd-insert uuid=$vm cd-name=<vates-autopilot-docking-....iso>
xe vm-param-set uuid=$vm HVM-boot-params:order=dc platform:secureboot=false
xe vm-param-set uuid=$vm xenstore-data:vm-data/xo-docking/mode=diagnose \
xenstore-data:vm-data/xo-docking/keep-running=1
xe vm-start uuid=$vm
```
`diagnose` changes nothing in the guest; `dock` can be run again safely, see [XO side](#xo-side).
With `keep-running`, the console logs in as the live user (`user`, with `sudo`):
`/run/xo-docking/report.json` and `/run/xo-docking/xo-docking.log` hold the run, the guest disks can be
mounted, and `sudo xo-docking --mode diagnose --verbose` runs it again. A `diagnose` run also reads the
state of a Windows first boot offline (`os.firstboot`, with the end of `firstboot.log`).
Afterwards, shut the VM down, then put back what XO does after its runs:
```sh
xe vm-cd-eject uuid=$vm
xe vm-param-set uuid=$vm HVM-boot-params:order=c
xe vm-param-remove uuid=$vm param-name=xenstore-data param-key=vm-data/xo-docking/mode
xe vm-param-remove uuid=$vm param-name=xenstore-data param-key=vm-data/xo-docking/keep-running
# and each vm-data/xo-docking/out/ key listed by: xe vm-param-get uuid=$vm param-name=xenstore-data
```
Set `platform:secureboot=true` again if the guest used it.
### Modes
**`diagnose`** is the default. It only reads: filesystems are mounted read-only without journal
replay (`noload`, `norecovery`, `rescue=nologreplay`), and actions are reported as `planned`, which
shows exactly what `dock` would do.
**`dock`** applies the planned actions in order, unless a blocking error was found, in which case
nothing is modified. A failing critical action stops the run and the following actions are reported
as `skipped`.
### Layout
```
auto/config live-build options (Debian trixie, BIOS + UEFI hybrid ISO)
config/package-lists/ packages of the live system
config/hooks/normal/ enable the service; zero boot menu timeouts; boot menu titles
(the default hooks of live-build are linked by `lb config`)
config/bootloaders/splash.svg boot menu background (logo embedded in base64), 640x480 for
isolinux, scaled to 800x600 for grub
config/includes.chroot/ files copied into the live system
etc/systemd/system/xo-docking.service
usr/local/bin/xo-docking
usr/lib/xo-docking/xo_docking/ the tool (Python stdlib + python3-hivex)
scripts/build.sh build, check and version the ISO
scripts/check-iso.sh post-build checks on the ISO
tests/ unit tests (python3 -m unittest)
```
The tool depends only on Debian packages: no pip, no lockfile. Security updates come from rebuilding
the ISO.
### Build
On Debian trixie, as root, from this folder:
```sh
apt-get install live-build
scripts/build.sh # produces dist/vates-autopilot-docking-<version>.iso and its .sha256
lb clean # auto/clean also removes generated config files
```
`scripts/build.sh` writes the version (`YYYYMMDDTHHMMSSZ-<sha>`, with `-dirty` if tracked files are modified,
or `YYYYMMDDTHHMMSSZ-$XO_DOCKING_VERSION`) into the ISO, then runs `lb config`, `lb build` and `scripts/check-iso.sh`.
The build needs a privileged container (live-build mounts chroots), e.g.
`podman run --rm --privileged -v "$PWD:/src" -w /src debian:trixie scripts/build.sh` after installing
live-build in it.
### Development
```sh
python3 -m unittest discover -s tests -v # needs python3-hivex for the registry tests
```
The tool can also be run by hand from the live system, or from any Linux system with the guest disks
attached. Without `--service` it never powers off:
```sh
xo-docking --mode diagnose --verbose
cat /run/xo-docking/report.json /run/xo-docking/xo-docking.log
```
Kernel command line switches, for testing without XO: `xo-docking.mode=dock`,
`xo-docking.keep-running`, and `xo-docking.disable`, which skips the service. With `keep-running`, the
console logs in automatically as the live user (`user`, with `sudo`).

View File

@@ -1,63 +0,0 @@
# Works on GitHub Actions and Gitea Actions (which also reads .github/workflows).
#
# Gitea notes:
# - the build job needs a privileged container (live-build mounts chroots):
# set `container.privileged: true` in the act_runner configuration;
# - nodejs is installed explicitly because JavaScript actions (checkout,
# upload-artifact) run inside the job container there;
# - older Gitea releases only support actions/upload-artifact@v3.
name: build
on:
push:
branches: [main]
tags: ['v*']
pull_request:
schedule:
- cron: '0 3 * * 1' # weekly rebuild to pick up Debian security updates
workflow_dispatch:
defaults:
run:
shell: bash
jobs:
test:
runs-on: ubuntu-latest
container: debian:trixie
steps:
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends ca-certificates git nodejs python3 python3-hivex
- uses: actions/checkout@v4
- name: Unit tests
run: python3 -m unittest discover -s tests -v
iso:
needs: test
runs-on: ubuntu-latest
container:
image: debian:trixie
options: --privileged
steps:
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends \
ca-certificates git nodejs live-build squashfs-tools xorriso
- uses: actions/checkout@v4
- name: Build
id: version
run: |
# release builds take their version from the tag, others from the date and commit
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
export XO_DOCKING_VERSION="${GITHUB_REF_NAME#v}"
fi
scripts/build.sh
- uses: actions/upload-artifact@v4
with:
name: vates-autopilot-docking-${{ steps.version.outputs.version }}
path: dist/
retention-days: 30

View File

@@ -1,29 +0,0 @@
# default hooks, linked by `lb config` from /usr/share/live/build/hooks: the
# hooks of this project are regular files, named outside these patterns
config/hooks/live/
config/hooks/normal/[0-8]*
config/hooks/normal/90[0-2]0-remove-*
# what live-build and scripts/build.sh generate, see auto/clean
.build/
binary/
cache/
chroot/
local/
config/binary
config/bootstrap
config/build
config/chroot
config/common
config/source
build.log
build.status
binary.modified_timestamps
chroot.files
chroot.packages.*
live-image-*
config/includes.chroot/usr/lib/xo-docking/VERSION
# Python bytecode, written by the tests: it must not end up in the ISO either
__pycache__/
*.pyc

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,235 +0,0 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.
A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.
The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.
An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based on the Program.
To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.
A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see <http://www.gnu.org/licenses/>.

View File

@@ -1,606 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @xen-orchestra/autopilot-docking
> Live ISO that prepares a VM imported from another hypervisor to boot on XCP-ng
## Usage
VMs Auto-pilot docking prepares a VM migrated from VMware to boot on XCP-ng, before its guest OS ever
boots there. It is made of two parts:
- **a live ISO**, built from this folder: booted in the migrated VM, it inspects the guest disks and
applies offline what the guest needs on XCP-ng (Xen PV drivers in the initramfs, VMware tools
disabled, XCP-ng guest tools installed, network configuration carried over…), then powers the VM
off. It runs inside the target VM, so it sees the real XCP-ng virtual hardware;
- **the orchestration in xo-server** (`packages/xo-server/src/xo-mixins/vmware/`), enabled by the
`docking` option of the import from VMware, and `startVm` to follow the first boot.
XO and the ISO talk through `VM.xenstore_data`: XO writes the inputs before booting the ISO, the ISO
writes its state and a compact report, see [xenstore contract](#xenstore-contract).
### Life cycle of a migrated VM
Each step is a state of the VM, each transition the condition XO waits for. The VM boots 2 to 3 times
on the ISO, then once on its disk for Linux and twice for Windows (plus one reboot to enable Secure
Boot again, when the source had it).
```mermaid
flowchart TD
S0[["0 · import complete<br/>VM halted, start blocked<br/>snapshot: complete import from V2V"]]
S1["1 · boot on the ISO, diagnose<br/>XO: ISO in the CD drive, boot order dc,<br/>guest tools ISO as a read-only disk, Secure Boot off<br/>ISO: reads the disks, writes the report, powers off"]
S2["2 · XO moves the OS disk to userdevice 0"]
S3["3 · boot on the ISO, dock<br/>XO: nics (Linux), windows-nics and windows-reboot (Windows)<br/>ISO: applies the actions, writes report and log in the guest, powers off"]
S4["4 · XO: ISO ejected, tools disk detached, boot order restored,<br/>xenstore keys cleared, timeoffset (Windows),<br/>start unblocked"]
S5["5 · boot on the disk"]
L6["6L · Linux boots on the PV drivers<br/>SELinux relabel: one more reboot, by the guest<br/>kdump rebuilds its initrd"]
W6["6W · first boot, phase drivers<br/>installs the PV drivers and the agent<br/>out/firstboot = drivers:reboot, shuts down"]
W7["7W · XO starts the VM, phase finish<br/>static addresses from network.ini<br/>out/firstboot = done, first-boot service deleted"]
WF["XO stops the VM and boots the ISO in diagnose<br/>to read the state of the first boot offline"]
S8["8 · XO waits for the guest agent, 15 min at most"]
S9["9 · Secure Boot on, clean reboot<br/>without agent after it: Secure Boot off, VM started again"]
E0(["VM running on XCP-ng"])
E1(["VM halted and docked:<br/>Windows reboots by itself between its phases"])
E2(["VM halted: FIRST_BOOT_FAILED<br/>C:\xo-docking\firstboot.log"])
E3(["VM running on emulated devices, without agent"])
EF(["VM halted, not docked: reason in the task<br/>the snapshot of the import is the rollback"])
S0 -->|"docking option"| S1
S1 -->|"boot_disk_not_first, once"| S2
S2 --> S1
S1 -->|"other blocking finding, or no result in 30 min"| EF
S1 -->|"nothing blocking"| S3
S3 -->|"result = error"| EF
S3 -->|"result = ok or warning"| S4
S4 -->|"startVm off"| E1
S4 -->|"startVm on"| S5
S5 -->|"Linux"| L6
S5 -->|"Windows, build 14393 and later"| W6
S5 -->|"Windows before build 14393"| E3
L6 --> S8
W6 -->|"VM halted"| W7
W7 -->|"out/firstboot = done"| S8
W6 & W7 -->|"failed, halted again, or 45 min"| WF
WF --> E2
S8 -->|"agent up, Secure Boot on the source"| S9
S8 -->|"no Secure Boot on the source, or no agent in 15 min"| E0
S9 --> E0
classDef ok fill:#d4edda,stroke:#2e7d32,color:#1b3d1f
classDef halted fill:#fff3cd,stroke:#b58105,color:#4d3800
classDef failed fill:#f8d7da,stroke:#c62828,color:#4a1015
class E0,E3 ok
class E1 halted
class E2,EF failed
```
Boot by boot:
| Boot | On | Started by | What runs | Ends with |
| ---- | ---- | --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- |
| 1 | ISO | XO | `diagnose`: read-only inspection, planned actions, blocking findings, `rtc_timeoffset` and network of Windows | power off by the ISO |
| 1b | ISO | XO | `diagnose` again, only after XO moved the OS disk first (`boot_disk_not_first`) | power off by the ISO |
| 2 | ISO | XO | `dock`: the planned actions, in order; full report and log written in the guest | power off by the ISO |
| 3 | disk | XO (`startVm`) or the administrator | Linux: the guest itself, SELinux relabel if scheduled. Windows: phase `drivers` of the first boot, PV drivers and agent | Linux: running. Windows: shut down (`windows-reboot=xo`) or reboot (`guest`) |
| 4 | disk | XO (`windows-reboot=xo`), the guest otherwise | Windows only: phase `finish`, static addresses reapplied, first-boot service deleted | running, `out/firstboot = done` |
| 5 | disk | XO (`VM.clean_reboot`) | only when the source had Secure Boot and the pool has its UEFI certificates | running, agent reporting |
A failed docking never fails the import: the data is there, and the snapshot taken at the end of the
import is the rollback. The task of the import carries the detail of each run and of the first boot.
### Limits
The VM is imported but not docked, and left halted, when:
- the import is partial: the source was running and `Stop source` was not set. The task says so, the
VM can be docked once a later import completes it;
- no docking ISO is deployed in an ISO SR of the pool (`DOCKING_ISO_NOT_FOUND`);
- its system volume is encrypted (LUKS, BitLocker);
- it is a Windows before 7 SP1 / 2008 R2 SP1, on dynamic disks, hibernated, with Fast Startup, or not
cleanly shut down;
- no Linux or Windows is found on its disks (BSD...).
The task of the import gives the reason.
It is docked, with less than a full migration, when:
- it is a Windows before build 14393: no PV drivers and no management agent;
- it is a Windows with several NICs whose source did not report its addresses (not running, or without
VMware Tools): its static addresses are not carried over, the adapters use DHCP;
- the source had Secure Boot: it is enabled again only when XO follows the first boot (`startVm`);
- the guest is in [partially supported](#partially-supported).
Only imports from VMware use it.
### Supported guests
"Validated" means: installed on ESXi 8 with the VMware defaults (PVSCSI or LSI SAS, vmxnet3 or
e1000e, open-vm-tools / VMware Tools), imported with XO, docked by the ISO, then booted on XCP-ng with
no manual step. The guest runs on PV disk and NIC, and XO gets the OS, the agent version, the memory
and the IP addresses.
#### Fully supported
| Family | Validated | Initramfs | Guest agent | Network configuration |
| ----------- | -------------------------------------------------------------------------- | ------------------------- | -------------------------------- | ------------------------------------- |
| Debian | 10, 11, 12 (BIOS), 13 (BIOS, UEFI) | initramfs-tools | `.deb` | ifupdown |
| Ubuntu | 22.04 (UEFI), 24.04 (BIOS), 26.04 desktop (UEFI) | initramfs-tools | `.deb` | netplan, NetworkManager |
| RHEL family | CentOS 7.3, Rocky 8.10, 9.8, 10.2, Oracle Linux 9.8 (UEK and RHCK kernels) | dracut | `.rpm` | ifcfg, NetworkManager |
| SUSE | SLES 16.0, openSUSE Leap 15.6 (BIOS, UEFI), SLES 15 SP7 (UEFI) | dracut | `.rpm` | NetworkManager, wicked |
| Arch | rolling (BIOS) | mkinitcpio (experimental) | generic `.tgz` | systemd-networkd |
| Windows | Server 2016, 2019, 2022, 2025 | n/a | PV drivers 9.x, management agent | captured, reapplied by the first boot |
Configurations validated across these guests: BIOS and UEFI, Secure Boot off on the source, SELinux
enforcing (relabel), btrfs with snapper, LVM over three disks with the root LV spanning two, `/boot`
and `/usr` each on their own disk, fstab by UUID, by `dm-uuid` and by YaST device ID, two NICs with
static addresses on both (Linux, and Windows Server 2022 with `windows-nics`: each address on its VIF,
DNS queries seen on the first one), Windows static address with its DNS reapplied by the first boot
(Server 2019 and 2022), DHCP (Ubuntu 22.04 gets the same address on XCP-ng as on VMware: same MAC,
same lease), kdump enabled.
RHEL itself is not tested (no subscription on the test bench): Rocky Linux and Oracle Linux (RHCK
kernel) share its kernel, dracut, rpm and NetworkManager; the RHEL subscription is left untouched.
#### Partially supported
| Guest | What works | What is missing |
| ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Windows before build 14393: 7 SP1, 8.1 and Server 2012 R2 validated, 2008 R2 expected the same | Offline changes; the VM boots on the emulated IDE disk and the emulated NIC (inbox driver), with the VMware MAC: DHCP and MAC reservations keep working, a static IP is reapplied by the first boot | The XCP-ng 9.x PV drivers do not support these versions (`pv_drivers_unsupported`): by default the first boot installs no package. No PV drivers, no management agent, so no `out/firstboot` in xenstore and XO does not follow the first boot: its state is only readable offline (`diagnose`, `os.firstboot`) |
| Transactional systems with a read-only root (SL Micro 6.2) | Nothing is written offline. The VM boots on PV devices when its initramfs already has the Xen drivers, which the ISO checks (`initramfs_xen_drivers_present`, true for the generic SL Micro image) | The guest agent and the other offline changes are listed in `readonly_root`, to do after the first boot with `transactional-update`. An initramfs without the Xen drivers is blocking (`readonly_root_initramfs`): rebuild it on the source first |
| Linux with neither systemd nor a package manager supported by the guest tools | Initramfs, fstab, VMware tools, network | No guest agent (`guest_agent_unsupported`): install it after the first boot |
| Linux without systemd-udevd | Initramfs, fstab, VMware tools, agent | Interface names are not pinned (`linux_interface_names`) |
| Linux with a kernel without `xen-blkfront` | Boots on the emulated IDE disk (`no_xen_blkfront`) | No PV disk |
#### Not supported
- Windows before 7 SP1 / 2008 R2 SP1 (`unsupported_windows_version`).
- Encrypted system volume, LUKS root or BitLocker on the Windows volume: the ISO has no key, so it
reports the volume (`luks_volume`, `bitlocker_volume`) and finds no system to change
(`no_os_found`, blocking). Decrypt it on the source before the migration. An encrypted data volume,
next to a system in clear, is reported and left untouched: the guest is docked, the volume itself is
not tested.
- Windows dynamic disks, hibernated Windows or Fast Startup: blocking.
- Windows volume not cleanly shut down (`ntfs_unclean`, blocking): the source was powered off, or
imported while running and then powered off. ntfs-3g only mounts such a volume read-only, so nothing
could be written. The source must be shut down from the guest (VMware Tools, which `Stop source`
does), not powered off.
- Anything else (BSD, ...): `no_os_found`. When MD RAID or LVM could not be activated, the report
also has a `storage_activation_failed` warning per failed command, with its error: the system may
be on them.
### What the ISO does
Detection never modifies anything: it records findings and plans actions, each with a stable `code`.
`dock` applies them in order. Non-critical actions only produce a warning when they fail.
#### All Linux guests
| Action | Code | What it does |
| ------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Initramfs | `initramfs_xen_drivers` | Rebuilds it with the guest's own tool in a chroot, with `xen-blkfront`, `xen-netfront` and the emulated IDE drivers. Critical: a PVHVM kernel unplugs the emulated disks at boot whenever it has blkfront support, so without the module in the initramfs the root disk disappears |
| fstab | `fstab_stable_names` | Rewrites `/dev/sdX`, and hardware `by-id` / `by-path` names of the root disk, to `UUID=`. In place (inode and SELinux label kept), backup in `fstab.xo-docking.bak`. Content-derived names (`dm-uuid-`, `lvm-pv-uuid-`, `md-uuid-`) are stable and left alone |
| VMware tools | `disable_vmware_tools` | Disables the open-vm-tools / vmware-tools units (`vmtoolsd`, `vgauth`, `vgauthd`). Masked units are left alone |
| Network | `linux_pin_interface_names` | Interface names change on XCP-ng (`ens192` → `enX0`). For each old name found in the network configuration, writes `/etc/systemd/network/10-xo-docking-<name>.link`, which gives the name back to the NIC with the same MAC, before the initramfs is rebuilt. The guest configuration (ifupdown, ifcfg, NetworkManager, netplan, networkd) then applies unchanged. Automatic with one NIC; with several, XO passes `nics`. Kernel names (`eth0`) do not change. The network configuration itself is never modified: static and DHCP interfaces stay as they are, and a DHCP reservation by MAC keeps working since the VIFs keep the VMware MACs |
| Guest agent | `install_guest_agent` | When neither `xe-guest-utilities` nor `xen-guest-agent` is there, installs `xe-guest-utilities` from the [guest tools disk](#guest-tools-disk) and enables `xe-linux-distribution` (not started) |
| kdump | `kdump_initrd_rebuild` | Deletes the kdump initrd built on VMware without `xen-blkfront`; kdump rebuilds it on XCP-ng at the next boot. Without this, the capture kernel does not find the disk |
| SELinux | `selinux_relabel` | When SELinux is enabled and anything was changed, schedules a relabel (`/.autorelabel`), which may cost one extra reboot |
#### Per family
- **Debian, Ubuntu**: initramfs-tools (`update-initramfs -u -k all`, modules appended to
`/etc/initramfs-tools/modules`). Agent: the `.deb`, installed with `dpkg -i` under a temporary
`policy-rc.d` that forbids starting services: in a chroot, `invoke-rc.d` would otherwise start the
daemon on the live system.
- **RHEL family** (RHEL, CentOS, Rocky, Alma...): dracut (`--add-drivers`, per kernel). Agent: the
`.rpm` (`rpm -U`; the scriptlets enable the unit and `systemctl` starts nothing in a chroot), or the
`legacy` `.rpm` with `chkconfig` when there is no systemd. kdump: `/boot/initramfs-*kdump.img`,
rebuilt by `kdumpctl`.
- **SUSE** (SLES, openSUSE): dracut. Root on btrfs with snapper: the default subvolume is mounted,
`subvol=` options of fstab are honoured. fstab by device ID (YaST "mount by device ID", older SLES)
is rewritten when it names the root disk: its entry tells which VMware name is that disk, and the
partitions follow. kdump: `/var/lib/kdump/initrd`, rebuilt by `mkdumprd`. Agent: the `.rpm`.
- **Arch and other systemd guests without dpkg or rpm**: mkinitcpio (drop-in
`/etc/mkinitcpio.conf.d/90-xo-docking.conf`, `mkinitcpio -P`), reported as experimental. Agent: the
generic `.tgz`, extracted by the ISO following the guest's own links (`/usr/sbin` → `usr/bin`), with
the systemd unit of the guest tools ISO.
- **Transactional systems** (SL Micro, root on a read-only btrfs snapshot, or `/` mounted `ro` in
fstab with `transactional-update` present): the root is never written offline. Instead of
rebuilding the initramfs, the ISO lists it (`lsinitramfs`) and checks that `xen-blkfront` and
`xen-netfront` are in it for every kernel.
#### Windows (7 SP1 / 2008 R2 SP1 and later)
Offline, from the ISO, in every control set so that "Last Known Good Configuration" boots too. The
SYSTEM hive is backed up as `SYSTEM.xo-docking.bak` before being written (`commit_registry`).
| Action | Code | What it does |
| --------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Boot drivers | `enable_ide_boot` | `atapi`, `intelide`, `pciide` start at boot, `StartOverride` removed, so Windows boots from the emulated IDE disk |
| VMware services | `disable_vmware_services` | Disables them (Start=4), found by name and by rule (image path under `\VMware\VMware Tools\`, or a `vm*` kernel driver described as VMware). Conservative: other VMware/Broadcom products (Carbon Black...) are left alone |
| Network | `clear_old_static_ip` | Captures the TCP/IP configuration of the old PCI adapters in the report (`network`), then switches the ones with a static address to DHCP; otherwise Windows refuses the same address on the new adapter. Adapters already in DHCP are left untouched. The new PV adapter starts in DHCP with the VMware MAC, so a DHCP reservation by MAC applies to it as before. The static configuration (addresses, masks, gateways, DNS) is written to `\xo-docking\network.ini` for the first boot, keyed by the MAC of the VIF: automatic with a single static adapter on a single NIC, from `windows-nics` otherwise (`windows_static_ip_unmapped` when an adapter has no MAC) |
| First boot | `install_firstboot` | Installs the temporary service `XoDockingFirstboot` and `\xo-docking\` (`firstboot.ps1`, `config.ini`), and copies the packages matching `windows-msi-patterns` from the guest tools disk into `\xo-docking\packages\` |
Also reported: `os.rtc_timeoffset` (seconds, which XO sets as `platform:timeoffset`: Windows keeps the
RTC in local time and XCP-ng starts it at UTC), the state of a previous first boot (`os.firstboot`,
`windows_firstboot_incomplete` with the end of its log), and `pv_drivers_unsupported` below build 14393. Below that build, the default `windows-msi-patterns` is empty: the XCP-ng 9.x MSI would fail and
stop the first boot in its `drivers` phase, before `finish` reapplies the static IP. Patterns given
explicitly by XO are always used.
##### Windows first boot
`firstboot.ps1` runs at each boot as LocalSystem until its phases are done, then deletes its service.
It is compatible with PowerShell 2.0. Before any phase, it restricts `C:\xo-docking` to SYSTEM and the
administrators (written through ntfs-3g, it is writable by every user), and removes the `Run` entries
that start a program from `\VMware\VMware Tools\` ("VMware User Process"), which crashes without its
services.
| Phase | `standard` | `full` | What it does | Ends with |
| --------- | ---------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------- |
| `drivers` | yes | yes | installs the packages matching `windows-msi-patterns` from `\xo-docking\packages\`, in order, then tries the VMware Tools MSI uninstall | reboot |
| `cleanup` | | yes | removes VMware driver packages from the driver store (`pnputil -f -d`, provider read from the INF), the VMware Tools directories, and the VMware-signed `vm*.dll` / `vm*.sys` / `pvscsi.sys` from System32 | reboot |
| `finish` | yes | yes | applies `network.ini` to the adapters with these MACs (WMI `EnableStatic`, `SetGateways`, `SetDNSServerSearchOrder`). `full` only: removes the ghost VMware PCI devices (vendor 15AD) with `pnputil /remove-device`, on Windows 10 2004 / Server 2022 and later. Then reports `done` | service deleted |
With `windows-reboot=xo`, each reboot of the table is a shut down instead (`shutdown /s`): XO, which
follows the first boot, starts the VM again. Before the PV drivers are loaded the guest cannot tell it
anything, so a stopped VM is the signal. Once they are loaded, the script also writes
`<phase>:reboot` before shutting down. The default, `guest`, is for a VM started by hand: it must never
be left stopped in the middle of its first boot. XO sets `xo` when it follows the first boot
(`startVm`) and the PV drivers support the guest.
A failing phase is retried at the next boot, up to 3 attempts, then the script gives up
(`<phase>:failed`) and deletes its service. The log is in `\xo-docking\firstboot.log`, next to the MSI
installation logs.
`standard` is the default and non-destructive: the VMware components stay on disk, disabled. `full`
follows [vmwremover](https://github.com/virtio-win/vmwremover), except that vsock is kept (removing it
means editing the Winsock catalog) and devcon.exe is not used (not redistributable): ghost devices stay
where pnputil cannot remove them, which is harmless since the old static addresses were cleared.
#### UEFI (all guests)
The NVRAM on XCP-ng is new, so the boot entries created on VMware are lost. The ISO does not write
NVRAM: it makes sure `\EFI\BOOT\BOOTX64.EFI` exists on the ESP (`efi_fallback_loader`). For Linux it
copies shim, GRUB and grub.cfg from the vendor directory, like `grub-install --force-extra-removable`;
for Windows, `bootmgfw.efi`.
#### Detected but not changed
- **Interface names that could not be pinned**: several NICs without `nics`, or no systemd-udevd
(`linux_interface_names`).
- **Unstable device names left**: hardware by-id/by-path names of other disks than the root one
(`fstab_unstable_names`), `resume=` or `root=` with such a name in `/etc/default/grub`
(`grub_unstable_resume`, `grub_unstable_root`): the initramfs waits for that device at every boot.
- **Firmware mismatches** (Windows on GPT with BIOS, UEFI without ESP).
- **OS not on the first disk** (`boot_disk_not_first`, blocking: nothing is modified). XCP-ng boots
the first disk, so such a VM does not boot. XO moves the VBD of `data.os_disk` to `userdevice` 0 and
runs the ISO again. Validated with Debian 13 (data disk first, OS on the second disk).
- **Encryption, dynamic disks, hibernation**: see [not supported](#not-supported).
#### Left to the administrator
- **open-vm-tools / VMware Tools** stay installed, disabled (`standard` cleanup on Windows). Remove
them when convenient.
- **Secure Boot**: off during the ISO runs (the ISO is not signed). With `startVm`, XO turns it back on
after the first boot if the source had it and the pool certificates are installed, and turns it off
again if the VM does not boot with it: validated on Ubuntu 22.04, SLES 16.0 and Windows Server 2022.
Without `startVm`, enable it again once the VM has booted.
- **SCC / RHN registrations**: not tested.
### XO side
The orchestration is the [life cycle](#life-cycle-of-a-migrated-vm) above. Details that matter:
- **The ISO** is the most recent `vates-autopilot-docking-*.iso` (by name, which starts with the build date and time) of an
ISO SR of the pool, inserted in the CD drive of the VM (created when it has none) and ejected after
the runs. `DOCKING_ISO_NOT_FOUND` when there is none.
- **`platform:secureboot=false` is mandatory for every UEFI VM** on each boot on the ISO, even when XO
shows `secureBoot: false`: an absent key means `auto` on XCP-ng 8.3. Symptom: the VM lands in the
UEFI Shell (`Access Denied` on `\EFI\BOOT\BOOTX64.EFI`) and nothing appears in xenstore. A signed ISO
would remove this.
- **`nics`** is built from the networks of the VMX, `ens<ethernetN.pciSlotNumber>=<MAC>` for each NIC:
the predictable name the guest gave it on VMware (`pciSlotNumber` 192 → `ens192`). With it, a
migration with several NICs needs no manual step.
- **`windows-nics`** matches the addresses the VMware tools reported while the source was running (MAC
→ addresses) with the `network` of the diagnose report (GUID → addresses). A Windows source with
several NICs which reported nothing (not running, no VMware tools) keeps DHCP on all its adapters,
with a warning.
- **Running the ISO again** (retry after a failure or a timeout) is safe and validated on Rocky 9 and
Windows Server 2022: on Linux the agent is not reinstalled and fstab and the pinned names stay as
they are (the initramfs is rebuilt and a relabel scheduled again); on Windows, a first boot that
already ended in `done` with the PV drivers installed is not scheduled again
(`windows_firstboot_already_done`). But the old adapters were switched to DHCP by the first run, so
the second report has no static configuration in `network`: the `network` of the first report is the
one to keep.
- **Stale keys**: `VM.xenstore_data` persists after the halt and `vm.set` merges keys, so before each
boot on the ISO XO deletes every input it does not set and the whole `out/` subtree, and deletes all
of them after the last run. Otherwise it may read a `done` from a previous run. The ISO also deletes
`out/` when it starts, but too late to protect XO.
- **Timeouts**: 30 minutes for a run of the ISO, 45 minutes for the Windows first boot, 15 minutes for
the guest agent to report after the first boot. XO does not wait for an agent the docking could not
install (`guest_agent_missing`, `guest_agent_unsupported`, failed `install_guest_agent`): it warns
right away.
#### Not done by XO
- The fallback for the adapters left in `windows_static_ip_unmapped` (`VIF.configure_ipv4` /
`VIF.configure_ipv6` once the agent is up, validated by hand on Windows Server 2019 with agent
9.2.385: address and gateway applied within seconds, no DNS). They stay in DHCP, with a warning.
- `windows-cleanup=full`: XO always uses `standard`.
- Windows patches (`.msp`): `firstboot.ps1` runs `msiexec /i` on every package. Not needed with the
current tools ISO (its `XenTools-fix-9.2.351.msp` targets 9.2.351, the MSI installs 9.2.385). If a
future one needs it: `/p` by extension, after its MSI, 1642 (not applicable) as skipped.
#### Guest tools disk
The ISO takes the XCP-ng guest tools from the guest tools ISO every pool already has, so the docking
ISO stays generic and the tools match the host.
- **VDI**: the guest tools ISO of the pool (`VDI.is_tools_iso = true`, in the `XCP-ng Tools` SR); XAPI
accepts a VDI of an ISO SR in a disk VBD.
- **VBD**: `type=Disk`, `mode=RO`, `userdevice=25` (`xvdz`) when free, `bootable=false`. Not a CD:
xen-blkfront ignores CD drives on HVM guests, so the live system would not see it.
- **Recognition**: a whole disk with an `iso9660` or `udf` filesystem and a `Linux/` or `Windows/`
directory. It is never taken as a guest disk.
- **Optional**: without it the report says `"tools": null`, Linux gets `guest_agent_missing` and
Windows `windows_tools_disk_missing`; the Windows first boot then has no package to install, and
gives up after its attempts.
#### xenstore contract
Paths are relative to the domain (`/local/domain/<domid>/`).
Inputs, set by XO through `VM.xenstore_data` (keys must start with `vm-data/`):
| Key | Values | Default |
| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- |
| `vm-data/xo-docking/mode` | `diagnose`, `dock` | `diagnose` |
| `vm-data/xo-docking/windows-msi-patterns` | `;`-separated file patterns of the packages to install, in order (e.g. `xen-*.msi;managementagent-*.msi`) | `*.msi`, none below build 14393 |
| `vm-data/xo-docking/windows-cleanup` | `standard` or `full`, see [Windows first boot](#windows-first-boot) | `standard` |
| `vm-data/xo-docking/windows-reboot` | `guest` or `xo`: who restarts the VM between the phases of the first boot, see [Windows first boot](#windows-first-boot) | `guest` |
| `vm-data/xo-docking/nics` | Linux, `;`-separated `<old name>=<MAC>` (e.g. `ens192=00:0c:29:d4:bf:19;ens224=00:0c:29:d4:bf:23`). Only needed with several NICs | unset |
| `vm-data/xo-docking/windows-nics` | Windows, `;`-separated `<MAC>=<GUID>`: the VIF that gets the static configuration of the old adapter with this `guid` in `network` (e.g. `00:0c:29:94:80:0d={44FED271-EE81-4363-8CF0-893542F0734F}`). Only needed with several NICs | unset |
| `vm-data/xo-docking/keep-running` | `1` to stay up after the run (debugging) | unset |
Outputs, written by the ISO under `vm-data/` too: XAPI syncs the guest's `vm-data/` subtree back into
`VM.xenstore_data`, but not `data/`.
| Key | Content |
| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `vm-data/xo-docking/out/version` | ISO version (`YYYYMMDD-<sha>` or the release tag) |
| `vm-data/xo-docking/out/mode` | mode actually used |
| `vm-data/xo-docking/out/state` | `running`, then `done` (written last: the other keys are final) |
| `vm-data/xo-docking/out/result` | `ok`, `warning` or `error` |
| `vm-data/xo-docking/out/report` | compact JSON report, written raw (`xenstore-write -R`) |
| `vm-data/xo-docking/out/firstboot` | Windows, written by the first-boot script once the PV drivers are loaded: `<phase>:running`, `<phase>:reboot` (`windows-reboot=xo`, the VM shuts down), `<phase>:error`, `<phase>:failed` or `done`. A later `diagnose` run reads it offline and writes it too (`<phase>:pending` if it never started) |
oxenstored refuses guest values over 2048 bytes (`quota-maxsize`), so the compact report is limited
to 1900 bytes. What XO never reads goes first: `os` is reduced to `kind`, `name`, `rtc_timeoffset` and
`firstboot`, `network` to the static adapters with their `guid`, `dhcp` and `addresses`. Then the
details of the findings, the info-level findings and the messages. Last, findings and actions are
dropped one by one, those XO acts on last: errors, failed actions, and the codes in `XO_CODES` of
`report.py` (`boot_disk_not_first`, `guest_agent_missing`, `guest_agent_unsupported`,
`install_firstboot`, `install_guest_agent`, `pv_drivers_unsupported`, `windows_firstboot_incomplete`,
`windows_static_ip_unmapped`), info ones included. In dock mode the full report and the log are also
written into the guest: `/var/log/xo-docking/` on Linux, `\xo-docking\` on Windows.
```json
{
"version": "20261012T084512Z-1a2b3c4d",
"mode": "diagnose",
"result": "warning",
"firmware": "uefi",
"disks": ["xvda", "xvdb"],
"tools": "xvdz",
"os": {
"kind": "linux",
"name": "Debian GNU/Linux 12 (bookworm)",
"id": "debian",
"version": "12",
"root": "/dev/xvda2",
"kernels": ["6.1.0-18-amd64"],
"initramfs": "initramfs-tools",
"guest_agent": "xe-guest-utilities_7.30.0-19_amd64.deb"
},
"network": [],
"findings": [
{ "level": "warning", "code": "linux_interface_names", "message": "...", "data": { "interfaces": ["ens192"] } }
],
"actions": [{ "code": "initramfs_xen_drivers", "status": "planned", "description": "..." }]
}
```
For Windows, `network` lists the TCP/IP configuration of the old adapters (`adapter`, `vmware`,
`dhcp`, `addresses`, `masks`, `gateways`, `dns`).
### Troubleshooting
Look at the task of the import, then at the files in the guest.
xenstore only helps while the ISO or the Windows first boot runs: XO deletes its keys after its runs.
#### The task of the import
Each step is a subtask, with its own status: `attaching the docking ISO`, a run of the ISO,
`releasing the docking ISO`, then for the first boot `starting the VM`, on Windows
`<phase> phase of the Windows first boot` and `restarting the VM after the <phase> phase of its first boot`,
and `waiting for the guest agent`. A phase of the Windows first boot fails when it does not report in
time, gives up, or stops the VM once more.
Each run of the ISO is a subtask, `docking ISO: diagnose` or `docking ISO: dock`, whose log carries the
version of the ISO, the result, the guest OS, then one line for each action (`action <code>: <status>`)
and each finding (`finding <level> <code>: <message>`). In a `dock`
run, a failed action and a finding above `info` are warnings of the task; a `diagnose` run is only a
plan, none of its lines is a warning.
| Line of a run of the ISO | Meaning |
| --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
| `docking ISO <version>: <mode> ok`, `warning` or `error` | result of the run |
| `action <code>: <status> (<error>)` | `planned` (diagnose only), `applied`, `failed`, or `skipped` after a failed critical action. The codes are in [what the ISO does](#what-the-iso-does) |
| `finding <level> <code>: <message>` | an `error` is blocking |
| `details were dropped from the report to fit in xenstore` | the full report is in the guest |
A blocking finding stops the docking after `diagnose`: the guest is as imported. When Windows stops
between the phases of its first boot for XO to start it again (`windows-reboot=xo`) and XO no longer
follows it, the warning of the task says to start the VM again until `firstboot.log` says it is done.
A failure does
not fail the import: it is a warning of the task, `the guest could not be docked: <message>` or
`the first boot on XCP-ng failed: <message>`, with one of these codes in the log of xo-server:
| Code | Meaning | What to do |
| --------------------------- | ------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
| `DOCKING_ISO_NOT_FOUND` | no `vates-autopilot-docking-*.iso` in an ISO SR of the pool. Checked before any transfer: this one fails the import | deploy the ISO, then import again |
| `DOCKING_ISO_TIMEOUT` | no result within 30 minutes: most often the firmware did not boot the ISO | [run the ISO by hand](#running-the-iso-by-hand) and watch the console: the UEFI Shell with `Access Denied` means Secure Boot is on |
| `DOCKING_ISO_NO_RESULT` | the VM halted without the ISO writing `done`, or without a report | run the ISO by hand with `keep-running`, read `/run/xo-docking/xo-docking.log` |
| `DOCKING_OS_DISK_NOT_FOUND` | the ISO found the OS on a disk the VM does not have | check the disks of the VM |
| `DOCKING_BLOCKED` | a blocking finding in `diagnose`: nothing was changed | the finding is in the task of the `diagnose` run, see [limits](#limits) |
| `DOCKING_FAILED` | the `dock` run failed: the guest may be partly changed | the files in the guest; revert to the snapshot of the import to start again |
| `FIRST_BOOT_FAILED` | the Windows first boot did not end: XO stopped the VM and read its state offline | `C:\xo-docking\firstboot.log` |
The snapshot of the import is `after complete import from V2V`, or `complete import from V2V` when the
source was stopped by the import (`Stop source`).
Warnings without failure: no guest tools ISO in the pool (no guest agent), static addresses not
carried over (`windows_static_ip_unmapped`), guest agent not reporting within 15 minutes of the first
boot, no guest agent installed by the docking, Secure Boot left off.
#### xenstore
XO deletes every `vm-data/xo-docking/` key before each run of the ISO and after the last one. After the
docking, only `vm-data/xo-docking/out/firstboot` can be there: the Windows first boot writes it once the
PV drivers are loaded, so it holds the last state it could report. XO deletes it too once the first
boot is done: it is only left when XO did not follow the
first boot to its end. From the pool master:
```sh
xe vm-param-get uuid=<VM UUID> param-name=xenstore-data param-key=vm-data/xo-docking/out/firstboot
```
During a run of the ISO, all the [outputs](#xenstore-contract) are there:
`xe vm-param-get uuid=<VM UUID> param-name=xenstore-data`.
#### The files in the guest
The report in xenstore is short on purpose (codes always, details when they fit). The details stay in
the VM: `dock` writes the full report and the full log of the ISO into the guest, next to what the
first boot produces. A `diagnose` run writes nothing.
| Guest | Files |
| ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Linux | `/var/log/xo-docking/report.json`, `/var/log/xo-docking/xo-docking.log`; backups next to the changed files (`/etc/fstab.xo-docking.bak`, `/etc/initramfs-tools/modules.xo-docking.bak`) |
| Windows | `C:\xo-docking\report.json`, `xo-docking.log` (ISO), `firstboot.log` (first boot, each phase and the static IP configuration), `install-*.log` (MSI), `network.ini`; `SYSTEM.xo-docking.bak` next to the SYSTEM hive |
When the guest does not boot, reach them by [running the ISO by hand](#running-the-iso-by-hand) with
`keep-running`, or by attaching the disk to another VM.
#### Running the ISO by hand
To watch a run on the console, or to reach the disks of a guest which does not boot. On the pool
master, with the VM halted:
```sh
vm=<VM UUID>
# the CD drive XO created; without one: xe vm-cd-add uuid=$vm cd-name=<ISO name> device=3
xe vm-cd-insert uuid=$vm cd-name=<vates-autopilot-docking-....iso>
xe vm-param-set uuid=$vm HVM-boot-params:order=dc platform:secureboot=false
xe vm-param-set uuid=$vm xenstore-data:vm-data/xo-docking/mode=diagnose \
xenstore-data:vm-data/xo-docking/keep-running=1
xe vm-start uuid=$vm
```
`diagnose` changes nothing in the guest; `dock` can be run again safely, see [XO side](#xo-side).
With `keep-running`, the console logs in as the live user (`user`, with `sudo`):
`/run/xo-docking/report.json` and `/run/xo-docking/xo-docking.log` hold the run, the guest disks can be
mounted, and `sudo xo-docking --mode diagnose --verbose` runs it again. A `diagnose` run also reads the
state of a Windows first boot offline (`os.firstboot`, with the end of `firstboot.log`).
Afterwards, shut the VM down, then put back what XO does after its runs:
```sh
xe vm-cd-eject uuid=$vm
xe vm-param-set uuid=$vm HVM-boot-params:order=c
xe vm-param-remove uuid=$vm param-name=xenstore-data param-key=vm-data/xo-docking/mode
xe vm-param-remove uuid=$vm param-name=xenstore-data param-key=vm-data/xo-docking/keep-running
# and each vm-data/xo-docking/out/ key listed by: xe vm-param-get uuid=$vm param-name=xenstore-data
```
Set `platform:secureboot=true` again if the guest used it.
### Modes
**`diagnose`** is the default. It only reads: filesystems are mounted read-only without journal
replay (`noload`, `norecovery`, `rescue=nologreplay`), and actions are reported as `planned`, which
shows exactly what `dock` would do.
**`dock`** applies the planned actions in order, unless a blocking error was found, in which case
nothing is modified. A failing critical action stops the run and the following actions are reported
as `skipped`.
### Layout
```
auto/config live-build options (Debian trixie, BIOS + UEFI hybrid ISO)
config/package-lists/ packages of the live system
config/hooks/normal/ enable the service; zero boot menu timeouts; boot menu titles
(the default hooks of live-build are linked by `lb config`)
config/bootloaders/splash.svg boot menu background (logo embedded in base64), 640x480 for
isolinux, scaled to 800x600 for grub
config/includes.chroot/ files copied into the live system
etc/systemd/system/xo-docking.service
usr/local/bin/xo-docking
usr/lib/xo-docking/xo_docking/ the tool (Python stdlib + python3-hivex)
scripts/build.sh build, check and version the ISO
scripts/check-iso.sh post-build checks on the ISO
tests/ unit tests (python3 -m unittest)
```
The tool depends only on Debian packages: no pip, no lockfile. Security updates come from rebuilding
the ISO.
### Build
On Debian trixie, as root, from this folder:
```sh
apt-get install live-build
scripts/build.sh # produces dist/vates-autopilot-docking-<version>.iso and its .sha256
lb clean # auto/clean also removes generated config files
```
`scripts/build.sh` writes the version (`YYYYMMDDTHHMMSSZ-<sha>`, with `-dirty` if tracked files are modified,
or `YYYYMMDDTHHMMSSZ-$XO_DOCKING_VERSION`) into the ISO, then runs `lb config`, `lb build` and `scripts/check-iso.sh`.
The build needs a privileged container (live-build mounts chroots), e.g.
`podman run --rm --privileged -v "$PWD:/src" -w /src debian:trixie scripts/build.sh` after installing
live-build in it.
### Development
```sh
python3 -m unittest discover -s tests -v # needs python3-hivex for the registry tests
```
The tool can also be run by hand from the live system, or from any Linux system with the guest disks
attached. Without `--service` it never powers off:
```sh
xo-docking --mode diagnose --verbose
cat /run/xo-docking/report.json /run/xo-docking/xo-docking.log
```
Kernel command line switches, for testing without XO: `xo-docking.mode=dock`,
`xo-docking.keep-running`, and `xo-docking.disable`, which skips the service. With `keep-running`, the
console logs in automatically as the live user (`user`, with `sudo`).
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

View File

@@ -1,7 +0,0 @@
#!/bin/sh
set -e
# keep lb build's exit status: a plain pipe into tee would return tee's
{ status=0; lb build noauto "${@}" 2>&1 || status=$?; echo "$status" > build.status; } | tee build.log
status="$(cat build.status)"
rm -f build.status
exit "$status"

View File

@@ -1,5 +0,0 @@
#!/bin/sh
set -e
lb clean noauto "${@}"
rm -f config/binary config/bootstrap config/chroot config/common config/source
rm -f build.log build.status

View File

@@ -1,22 +0,0 @@
#!/bin/sh
# live-build configuration of the docking ISO. Run through `lb config`.
set -e
lb config noauto \
--distribution trixie \
--archive-areas main \
--security true \
--updates true \
--apt-recommends false \
--binary-images iso-hybrid \
--bootloaders syslinux,grub-efi \
--uefi-secure-boot disable \
--debian-installer none \
--memtest none \
--firmware-binary false \
--firmware-chroot false \
--iso-volume XO-DOCKING \
--iso-application "VAPD : VMS Auto-pilot docking" \
--iso-publisher "Vates" \
--bootappend-live "boot=live components live-media=removable noswap quiet" \
"${@}"

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 89 KiB

View File

@@ -1,23 +0,0 @@
#!/bin/sh
# Nobody is at the console: boot the default entry immediately.
# Runs in the binary/ directory, after the boot loader configuration is generated.
# scripts/check-iso.sh verifies the result in CI, as templates change between
# live-build releases.
set -e
for f in isolinux/*.cfg; do
[ -f "$f" ] || continue
# syslinux counts in 1/10 s and "timeout 0" means "wait forever"
sed -i -E 's/^([[:space:]]*)timeout[[:space:]]+[0-9]+/\1timeout 10/I' "$f"
done
if [ -f isolinux/isolinux.cfg ] && ! grep -qiE '^[[:space:]]*timeout[[:space:]]' isolinux/isolinux.cfg; then
echo 'timeout 10' >> isolinux/isolinux.cfg
fi
for f in boot/grub/*.cfg; do
[ -f "$f" ] || continue
sed -i -E 's/^([[:space:]]*set[[:space:]]+timeout)=.*/\1=1/' "$f"
done
if [ -f boot/grub/grub.cfg ] && ! grep -q 'set timeout=' boot/grub/grub.cfg; then
sed -i '1i set timeout=1' boot/grub/grub.cfg
fi

View File

@@ -1,6 +0,0 @@
#!/bin/sh
# Enable the docking service in the live system.
set -e
chmod 0755 /usr/local/bin/xo-docking
systemctl enable xo-docking.service

View File

@@ -1,16 +0,0 @@
#!/bin/sh
# The name of the ISO is drawn in the boot menu background (config/bootloaders/splash.svg):
# remove the default menu titles, which would be drawn over it.
# Runs in the binary/ directory, after the boot loader configuration is generated.
set -e
# isolinux (BIOS): the title of the main menu only, the Utilities submenu keeps its own.
# The template ends it with a BEL character (^G, a beep when the menu shows): no `$` anchor
if [ -f isolinux/menu.cfg ]; then
sed -i -E '/^menu title Boot menu/d' isolinux/menu.cfg
fi
# grub (UEFI): live-build only clears it for its own default background
if [ -f boot/grub/live-theme/theme.txt ]; then
sed -i -E 's/^title-text:.*/title-text: ""/' boot/grub/live-theme/theme.txt
fi

View File

@@ -1,16 +0,0 @@
[Unit]
Description=Docking a VM migrated to XCP-ng
After=local-fs.target systemd-udev-settle.service
Wants=systemd-udev-settle.service
ConditionKernelCommandLine=!xo-docking.disable
[Service]
Type=oneshot
ExecStart=/usr/local/bin/xo-docking --service
StandardOutput=journal+console
StandardError=journal+console
# dracut on several kernels can be slow on an emulated disk
TimeoutStartSec=2h
[Install]
WantedBy=multi-user.target

View File

@@ -1 +0,0 @@
"""Docking VMs migrated from VMware so they boot on XCP-ng."""

View File

@@ -1,175 +0,0 @@
"""xo-docking: prepare a VM migrated from VMware to boot on XCP-ng.
Booted from the live ISO inside the migrated VM itself. Inputs come from
xenstore (set by XO), results go back to xenstore, then the VM powers off.
"""
from __future__ import annotations
import argparse
import logging
import sys
from pathlib import Path
from . import efi
from .blockdev import activate_storage, guest_disks, list_block_devices
from .detect import probe
from .linux import LinuxGuest, parse_nic_map
from .mounts import Mounts
from .report import Report
from .tools import find_tools
from .util import DockingError, kernel_cmdline_param, run
from .windows import WindowsGuest, parse_windows_nics
from .xenstore import Xenstore
log = logging.getLogger("xo-docking")
MODES = ("diagnose", "dock")
RUN_DIR = Path("/run/xo-docking")
VERSION_FILE = Path(__file__).resolve().parent.parent / "VERSION"
def read_version() -> str:
try:
return VERSION_FILE.read_text().strip() or "dev"
except OSError:
return "dev"
def setup_logging(verbose: bool) -> None:
RUN_DIR.mkdir(parents=True, exist_ok=True)
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stderr)
console.setLevel(logging.DEBUG if verbose else logging.INFO)
console.setFormatter(logging.Formatter("xo-docking: %(levelname)s %(message)s"))
logfile = logging.FileHandler(RUN_DIR / "xo-docking.log")
logfile.setLevel(logging.DEBUG)
logfile.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
log.addHandler(console)
log.addHandler(logfile)
def parse_args(argv):
parser = argparse.ArgumentParser(prog="xo-docking", description=__doc__)
parser.add_argument("--mode", choices=MODES, help="overrides xenstore and the kernel command line")
parser.add_argument("--service", action="store_true",
help="started by systemd: power off when done (unless keep-running is set)")
parser.add_argument("--verbose", action="store_true")
return parser.parse_args(argv)
def is_true(value: str | None) -> bool:
return value is not None and value.lower() in ("", "1", "true", "yes")
def execute(report: Report, mounts: Mounts, msi_patterns: list[str] | None, cleanup: str,
nic_map: dict[str, str], windows_nics: dict[str, str], reboot: str = "guest") -> None:
readonly = report.mode != "dock"
report.firmware = efi.firmware()
activation_failures = activate_storage(readonly)
devices = list_block_devices()
disks = guest_disks(devices)
report.disks = [d.name for d in disks]
if not disks:
report.error("no_disk", "the VM has no disk")
return
tools = find_tools(devices, mounts, report)
candidates = probe(devices, mounts, report)
if not candidates:
# the OS may be on RAID or LVM which could not be activated
for failure in activation_failures:
report.warning("storage_activation_failed", failure)
report.error("no_os_found", "no Linux or Windows installation found on the disks")
return
if len(candidates) > 1:
roots = [c.device.path for c in candidates]
report.error("multiple_os", "several operating systems found: " + ", ".join(roots), roots=roots)
return
candidate = candidates[0]
disk = next((d for d in disks if d.name == candidate.device.disk), disks[0])
if disk.name != disks[0].name:
# the firmware boots the first disk: the VM would not boot. Blocking, so
# that nothing is modified until XO puts the OS disk at userdevice 0.
report.error("boot_disk_not_first",
f"the OS is on {disk.name}, not on the first disk ({disks[0].name}): "
"move its VBD to userdevice 0 and run again", os_disk=disk.name, first_disk=disks[0].name)
if candidate.kind == "linux":
guest = LinuxGuest(candidate, devices, disks, mounts, report, readonly, nic_map, tools)
else:
guest = WindowsGuest(candidate, mounts, report, readonly, msi_patterns, cleanup, tools, windows_nics,
reboot=reboot)
try:
guest.inspect()
if not report.blocked:
efi.plan(report, mounts, devices, disk, guest.kind, guest.os_ids, readonly)
guest.finalize()
if report.mode == "dock":
if report.blocked:
report.info("dock_skipped", "blocking errors found: nothing was modified")
else:
report.apply_actions()
try:
guest.write_report(report)
except OSError as exc:
report.warning("guest_report_failed", f"cannot write the report in the guest: {exc}")
finally:
guest.close()
def main(argv=None) -> int:
args = parse_args(argv)
setup_logging(args.verbose)
xenstore = Xenstore.detect()
mode = args.mode or xenstore.config("mode") or kernel_cmdline_param("xo-docking.mode") or "diagnose"
# ";"-separated, installed in this order (e.g. drivers then management agent)
# None: default patterns, which depend on the Windows build
given_patterns = xenstore.config("windows-msi-patterns")
msi_patterns = [p for p in given_patterns.split(";") if p] if given_patterns else None
cleanup = xenstore.config("windows-cleanup") or "standard"
# "xo": the first boot stops the VM between its phases, XO starts it again
reboot = xenstore.config("windows-reboot") or "guest"
# old Linux interface name -> MAC, needed when the VM has several NICs
nic_map = parse_nic_map(xenstore.config("nics"))
# Windows: VIF MAC -> GUID of the old adapter whose static configuration it gets
windows_nics = parse_windows_nics(xenstore.config("windows-nics"))
keep_running = is_true(xenstore.config("keep-running")) or kernel_cmdline_param("xo-docking.keep-running") is not None
report = Report(mode, read_version())
xenstore.clear_status()
xenstore.status("version", report.version)
xenstore.status("mode", mode)
xenstore.status("state", "running")
log.info("xo-docking %s, mode %s", report.version, mode)
try:
if mode not in MODES:
raise DockingError("invalid_mode", f"unknown mode {mode!r}, expected one of {', '.join(MODES)}")
with Mounts(RUN_DIR / "mnt") as mounts:
execute(report, mounts, msi_patterns, cleanup, nic_map, windows_nics, reboot)
except DockingError as exc:
report.error(exc.code, str(exc))
except Exception as exc: # noqa: BLE001 - XO must always get a result
log.exception("unexpected error")
report.error("internal_error", f"{type(exc).__name__}: {exc}")
result = report.result()
(RUN_DIR / "report.json").write_text(report.full_json())
firstboot = report.os.get("firstboot")
if firstboot and mode == "diagnose":
# state of a previous Windows first boot, which could not send it itself
xenstore.status("firstboot", firstboot)
xenstore.status("report", report.compact_json())
xenstore.status("result", result)
xenstore.status("state", "done") # written last: XO can rely on the other keys
log.info("result: %s", result)
if args.service and not keep_running:
run(["systemctl", "poweroff", "--no-block"], check=False)
return 1 if result == "error" else 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,389 +0,0 @@
# XO docking - first boot on XCP-ng
#
# Started at every boot as LocalSystem by the temporary service XoDockingFirstboot
# (created offline by xo-docking) until all phases are done:
#
# standard: drivers -> finish
# full: drivers -> cleanup -> finish
#
# drivers install the Xen PV packages copied by xo-docking,
# uninstall VMware Tools (MSI),
# then reboot onto the PV drivers
# cleanup remove VMware driver packages and VMware Tools files, then reboot
#
# A reboot is a shut down when XO follows the first boot (reboot=xo in
# config.ini): XO starts the VM again, and knows where the first boot is.
# finish reapply the static IP configuration of the old adapters (network.ini),
# remove VMware ghost devices (full mode, Windows 10 2004+ only),
# report "done" to XO, delete the service
#
# Progress is written to C:\xo-docking\firstboot.log and, once the PV drivers are
# loaded, to xenstore (vm-data/xo-docking/out/firstboot = "<phase>:<state>").
#
# Must stay compatible with Windows PowerShell 2.0 (Windows 7 / Server 2008 R2):
# no ConvertFrom-Json, no CIM or PnpDevice cmdlets.
$ErrorActionPreference = 'Continue'
$Base = Split-Path -Parent $MyInvocation.MyCommand.Definition
$LogFile = Join-Path $Base 'firstboot.log'
$ServiceName = 'XoDockingFirstboot'
$Phases = @{ 'standard' = @('drivers', 'finish'); 'full' = @('drivers', 'cleanup', 'finish') }
$OkExitCodes = @(0, 1641, 3010) # success, success + reboot initiated, success + reboot required
# -- helpers -------------------------------------------------------------------
function Write-Log([string]$Message) {
$line = '{0} {1}' -f (Get-Date -Format 'yyyy-MM-ddTHH:mm:ss'), $Message
Add-Content -Path $LogFile -Value $line
}
function Read-Config {
$config = @{ 'msi_patterns' = '*.msi'; 'cleanup' = 'standard'; 'reboot' = 'guest'; 'max_attempts' = '3' }
$path = Join-Path $Base 'config.ini'
if (Test-Path $path) {
foreach ($line in Get-Content $path) {
if ($line -match '^\s*([^#=]+?)\s*=\s*(.*)$') { $config[$matches[1]] = $matches[2].Trim() }
}
}
return $config
}
function Get-State([string]$Name, [string]$Default) {
$path = Join-Path $Base $Name
if (Test-Path $path) { return [string](Get-Content $path | Select-Object -First 1) }
return $Default
}
function Set-State([string]$Name, [string]$Value) {
Set-Content -Path (Join-Path $Base $Name) -Value $Value
}
# xenstore through the WMI interface of the PV drivers (xeniface). Best effort:
# unavailable before the PV drivers are loaded, class names differ between builds
# (XenProject* for the XCP-ng 9.x drivers). Early in the boot the WMI provider
# may not be ready yet: $Attempts tries, 10 s apart.
function Send-Status([string]$Value, [int]$Attempts = 1) {
for ($attempt = 1; $attempt -le $Attempts; $attempt++) {
if ($attempt -gt 1) { Start-Sleep -Seconds 10 }
foreach ($class in @('XenProjectXenStoreBase', 'CitrixXenStoreBase', 'XenStoreBase')) {
try {
$base = Get-WmiObject -Namespace root\wmi -Class $class -ErrorAction Stop | Select-Object -First 1
if (-not $base) { continue }
$sessionId = $base.AddSession('xo-docking').SessionId
$sessionClass = $class -replace 'Base$', 'Session'
$session = Get-WmiObject -Namespace root\wmi -Query "select * from $sessionClass where SessionId=$sessionId"
$session.SetValue('vm-data/xo-docking/out/firstboot', $Value) | Out-Null
$session.EndSession() | Out-Null
Write-Log "status sent: $Value"
return
} catch {
continue
}
}
}
Write-Log "status not sent (xenstore WMI interface unavailable): $Value"
}
# The services are disabled offline, but the VMware user process is started at
# logon from the Run keys (SOFTWARE hive), and crashes without its services
# ("VMware Tools unrecoverable error"). The MSI uninstall removes it later.
function Remove-VmwareAutorun {
foreach ($key in @('HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Run',
'HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Run')) {
$values = Get-ItemProperty -Path $key -ErrorAction SilentlyContinue
if (-not $values) { continue }
foreach ($property in $values.PSObject.Properties) {
if ($property.Name -like 'PS*') { continue }
if ([string]$property.Value -like '*\VMware\VMware Tools\*') {
Remove-ItemProperty -Path $key -Name $property.Name -ErrorAction SilentlyContinue
Write-Log "removed autorun $($property.Name) from $key"
}
}
}
}
# xo-docking wrote this folder through ntfs-3g, which leaves it writable by every user of the guest, while
# its script and packages run as LocalSystem: restricted to SYSTEM and the administrators, by SID since the
# names of the groups are localized. At every run, before anything is read from the folder
function Protect-Base {
$output = & icacls.exe $Base /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' 2>&1 |
Out-String
if ($LASTEXITCODE -eq 0) {
# the files take the access of the folder
$output = & icacls.exe (Join-Path $Base '*') /reset /T /C 2>&1 | Out-String
}
if ($LASTEXITCODE -ne 0) {
Write-Log "warning: the access to $Base could not be restricted: $($output.Trim())"
}
}
function Remove-FirstbootService {
& sc.exe delete $ServiceName | Out-Null
Write-Log "service $ServiceName removed"
}
function Restart-Now([string]$Reason) {
if ($script:Config['reboot'] -eq 'xo') {
# XO starts the VM again. Only sent once the PV drivers are loaded: XO
# does not rely on it, a stopped VM is what it waits for
Send-Status "$($script:Phase):reboot"
Write-Log "shutting down, XO starts the VM again: $Reason"
& shutdown.exe /s /t 15 /c "XO docking: $Reason, XO starts the VM again"
exit 0
}
Write-Log "rebooting: $Reason"
& shutdown.exe /r /t 15 /c "XO docking: $Reason"
exit 0
}
function Stop-Attempt([string]$Reason) {
Write-Log $Reason
Send-Status "$($script:Phase):error"
if ($script:Attempt -ge [int]$script:Config['max_attempts']) {
Write-Log 'maximum number of attempts reached, giving up'
Send-Status "$($script:Phase):failed"
Remove-FirstbootService
} elseif ($script:Config['reboot'] -eq 'xo') {
# XO follows the first boot: the next attempt is now, not at a boot nobody does. A reboot and
# not a shut down, which XO would take for the end of the phase
Write-Log "rebooting for attempt $($script:Attempt + 1)"
& shutdown.exe /r /t 15 /c "XO docking: $Reason, retrying"
}
exit 1
}
# Copied from the guest tools disk by xo-docking. Only from there: what is installed as LocalSystem must come
# from a folder only SYSTEM and the administrators can write, see Protect-Base
function Find-Package([string]$Pattern) {
$local = Join-Path $Base 'packages'
if (-not (Test-Path $local)) { return $null }
$found = Get-ChildItem -Path $local -Filter $Pattern -ErrorAction SilentlyContinue | Select-Object -First 1
if ($found) { return $found.FullName }
return $null
}
function Invoke-Msi([string[]]$Arguments) {
$process = Start-Process -FilePath msiexec.exe -ArgumentList $Arguments -Wait -PassThru
return $process.ExitCode
}
function Test-VMwareSigned([string]$Path) {
$signature = Get-AuthenticodeSignature -FilePath $Path -ErrorAction SilentlyContinue
return ($signature -and $signature.SignerCertificate -and $signature.SignerCertificate.Subject -match 'VMware')
}
# Locked files are deleted at the next boot (MoveFileEx + MOVEFILE_DELAY_UNTIL_REBOOT)
Add-Type -ErrorAction SilentlyContinue -TypeDefinition @'
using System;
using System.Runtime.InteropServices;
public static class XoDockingFile {
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
public static extern bool MoveFileEx(string existing, string replacement, int flags);
}
'@
function Remove-FileOrSchedule([string]$Path) {
try {
Remove-Item -LiteralPath $Path -Force -ErrorAction Stop
Write-Log "deleted $Path"
} catch {
if ([XoDockingFile]::MoveFileEx($Path, $null, 4)) { Write-Log "scheduled for deletion at reboot: $Path" }
else { Write-Log "cannot delete $Path" }
}
}
# INF provider, resolving %Token% from the [Strings] section
function Get-InfProvider([string]$Path) {
$content = Get-Content -LiteralPath $Path -ErrorAction SilentlyContinue
$provider = ''
foreach ($line in $content) {
if ($line -match '^\s*Provider\s*=\s*(.+?)\s*$') { $provider = $matches[1]; break }
}
if ($provider -match '^%(.+)%$') {
$token = [regex]::Escape($matches[1])
foreach ($line in $content) {
if ($line -match "^\s*$token\s*=\s*`"?(.*?)`"?\s*$") { return $matches[1] }
}
}
return $provider.Trim('"')
}
# -- phases --------------------------------------------------------------------
function Invoke-DriversPhase {
# The XCP-ng 9.x MSI runs a managed custom action that needs .NET 4.x:
# without it, msiexec only says 1603.
$ndp = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\NET Framework Setup\NDP\v4\Full' -ErrorAction SilentlyContinue
if ($ndp) {
Write-Log ".NET Framework 4.x release $($ndp.Release)"
} else {
Write-Log 'warning: .NET Framework 4.x not found, the XCP-ng 9.x PV drivers will fail to install (1603)'
}
foreach ($pattern in $Config['msi_patterns'].Split(';')) {
if (-not $pattern) { continue }
$msi = Find-Package $pattern
if (-not $msi) { Stop-Attempt "no package matching $pattern in $Base\packages" }
$name = [IO.Path]::GetFileNameWithoutExtension($msi)
Write-Log "installing $msi"
$code = Invoke-Msi @('/i', "`"$msi`"", '/qn', '/norestart', '/l*v', "`"$(Join-Path $Base "install-$name.log")`"")
Write-Log "msiexec exit code $code"
if ($OkExitCodes -notcontains $code) { Stop-Attempt "installation of $msi failed" }
}
# VMware Tools: best effort, the uninstaller sometimes refuses to run outside
# VMware. Its services and drivers were already disabled offline.
$uninstallKeys = @(
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*',
'HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*'
)
$entries = Get-ItemProperty -Path $uninstallKeys -ErrorAction SilentlyContinue |
Where-Object { $_.DisplayName -like 'VMware Tools*' }
foreach ($entry in $entries) {
if ($entry.PSChildName -match '^\{[0-9A-Fa-f-]+\}$') {
$code = Invoke-Msi @('/x', $entry.PSChildName, '/qn', '/norestart')
Write-Log "VMware Tools removal exit code $code"
}
}
}
function Invoke-CleanupPhase {
# 1. VMware driver packages from the driver store. vsock is kept on purpose:
# removing it means editing the Winsock catalog, which this tool avoids.
foreach ($inf in Get-ChildItem -Path "$env:SystemRoot\INF" -Filter 'oem*.inf' -ErrorAction SilentlyContinue) {
$provider = Get-InfProvider $inf.FullName
if ($provider -notmatch 'VMware') { continue }
if ((Get-Content -LiteralPath $inf.FullName | Out-String) -match 'vsock') {
Write-Log "keeping $($inf.Name) (vsock)"
continue
}
$output = & pnputil.exe -f -d $inf.Name 2>&1 | Out-String
Write-Log "pnputil -f -d $($inf.Name): $($output.Trim())"
}
# 2. VMware Tools files. Whole directories, they only hold VMware Tools.
$directories = @(
"$env:ProgramFiles\VMware\VMware Tools",
"$env:ProgramData\VMware\VMware Tools",
"$env:ProgramData\VMware\VMware CAF",
"$env:ProgramData\VMware\VMware VGAuth"
)
if (${env:ProgramFiles(x86)}) { $directories += "${env:ProgramFiles(x86)}\VMware\VMware Tools" }
foreach ($directory in $directories) {
if (-not (Test-Path -LiteralPath $directory)) { continue }
Get-ChildItem -LiteralPath $directory -Recurse -Force -ErrorAction SilentlyContinue |
Where-Object { -not $_.PSIsContainer } |
ForEach-Object { Remove-FileOrSchedule $_.FullName }
Remove-Item -LiteralPath $directory -Recurse -Force -ErrorAction SilentlyContinue
}
# 3. System32: only files signed by VMware, vsock excluded (see above)
$system32 = Join-Path $env:SystemRoot 'System32'
$candidates = @()
$candidates += Get-ChildItem -Path $system32 -Filter 'vm*.dll' -ErrorAction SilentlyContinue
$candidates += Get-ChildItem -Path (Join-Path $system32 'drivers') -Filter 'vm*.sys' -ErrorAction SilentlyContinue
$candidates += Get-ChildItem -Path (Join-Path $system32 'drivers') -Filter 'pvscsi.sys' -ErrorAction SilentlyContinue
foreach ($file in $candidates) {
if ($file.Name -match 'vsock') { continue }
if (Test-VMwareSigned $file.FullName) { Remove-FileOrSchedule $file.FullName }
}
}
# Static IP configuration of the old (VMware) adapters, captured offline by
# xo-docking, applied to the new adapter with the same MAC. network.ini: one
# adapter per line, mac|addresses|masks|gateways|dns, lists comma-separated.
function Set-StaticNetwork {
$path = Join-Path $Base 'network.ini'
if (-not (Test-Path $path)) { return }
foreach ($line in Get-Content $path) {
if (-not $line.Trim()) { continue }
$fields = $line.Split('|')
$mac = $fields[0].ToUpper()
$adapter = $null
# the PV adapter can show up late after boot: wait up to one minute
for ($i = 0; $i -lt 12 -and -not $adapter; $i++) {
$adapter = Get-WmiObject Win32_NetworkAdapterConfiguration -Filter 'IPEnabled=TRUE' |
Where-Object { $_.MACAddress -eq $mac } | Select-Object -First 1
if (-not $adapter) { Start-Sleep -Seconds 5 }
}
if (-not $adapter) { Stop-Attempt "no network adapter with MAC ${mac}" }
$addresses = [string[]]@($fields[1].Split(',') | Where-Object { $_ })
$masks = [string[]]@($fields[2].Split(',') | Where-Object { $_ })
$gateways = [string[]]@($fields[3].Split(',') | Where-Object { $_ })
$dns = [string[]]@($fields[4].Split(',') | Where-Object { $_ })
# 0: done, 1: done, reboot required
$codes = @($adapter.EnableStatic($addresses, $masks).ReturnValue)
if ($gateways.Count) {
$metrics = [uint16[]]@($gateways | ForEach-Object { 1 })
$codes += $adapter.SetGateways($gateways, $metrics).ReturnValue
}
if ($dns.Count) { $codes += $adapter.SetDNSServerSearchOrder($dns).ReturnValue }
Write-Log "static IP on ${mac} ($($adapter.Description)): $($fields[1]) gateway $($fields[3]) dns $($fields[4]), results $($codes -join ',')"
foreach ($code in $codes) {
if ($code -gt 1) { Stop-Attempt "static IP configuration of ${mac} failed ($code)" }
}
}
}
function Invoke-FinishPhase {
Set-StaticNetwork
if ($Config['cleanup'] -eq 'full') {
# Ghost VMware PCI devices (vendor 15AD). pnputil /remove-device exists
# since Windows 10 2004 (build 19041); older versions keep them, which is
# harmless since the old static IPs were cleared offline.
$build = [int](Get-WmiObject Win32_OperatingSystem).BuildNumber
if ($build -ge 19041) {
$enum = 'HKLM:\SYSTEM\CurrentControlSet\Enum\PCI'
foreach ($device in Get-ChildItem -Path $enum -ErrorAction SilentlyContinue |
Where-Object { $_.PSChildName -like 'VEN_15AD*' }) {
foreach ($instance in Get-ChildItem -Path $device.PSPath -ErrorAction SilentlyContinue) {
$id = "PCI\$($device.PSChildName)\$($instance.PSChildName)"
$output = & pnputil.exe /remove-device $id 2>&1 | Out-String
Write-Log "pnputil /remove-device ${id}: $($output.Trim())"
}
}
} else {
Write-Log "build ${build}: ghost devices left in place"
}
}
}
# -- main ------------------------------------------------------------------------
Protect-Base
$Config = Read-Config
$Plan = $Phases[$Config['cleanup']]
if (-not $Plan) { $Plan = $Phases['standard'] }
$Phase = Get-State 'phase' $Plan[0]
if ($Phase -eq 'done') {
# a previous run finished but could not delete the service
Remove-FirstbootService
exit 0
}
$Attempt = [int](Get-State 'attempts' '0') + 1
Set-State 'attempts' $Attempt
Write-Log "phase $Phase, attempt $Attempt ($($Config['cleanup']) cleanup)"
Send-Status "${Phase}:running"
Remove-VmwareAutorun
$index = [array]::IndexOf($Plan, $Phase)
if ($index -lt 0) { Stop-Attempt "unknown phase $Phase" }
$next = $null
if ($index -lt $Plan.Length - 1) { $next = $Plan[$index + 1] }
switch ($Phase) {
'drivers' { Invoke-DriversPhase }
'cleanup' { Invoke-CleanupPhase }
'finish' { Invoke-FinishPhase }
}
if ($next) {
Set-State 'phase' $next
Set-State 'attempts' '0'
Restart-Now "$Phase done"
}
Set-State 'phase' 'done'
Set-Content -Path (Join-Path $Base 'firstboot.done') -Value (Get-Date -Format 's')
Send-Status 'done' -Attempts 6 # the signal XO waits for
Remove-FirstbootService
Write-Log 'all phases done'

View File

@@ -1,119 +0,0 @@
"""Block device inventory (lsblk) and storage stack activation."""
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from .util import DockingError, run
LSBLK_COLUMNS = "NAME,PATH,TYPE,FSTYPE,UUID,PARTUUID,LABEL,PARTTYPE,PTTYPE,SIZE,RM"
IGNORED_DISK_PREFIXES = ("loop", "sr", "zram", "ram", "fd")
# a whole disk holding an ISO filesystem is a medium, not a guest disk: the
# guest tools ISO attached read-only by XO (see tools.py)
MEDIA_FSTYPES = {"iso9660", "udf"}
ESP_PARTTYPES = {"c12a7328-f81f-11d2-ba4b-00a0c93ec93b", "0xef"}
BIOS_BOOT_PARTTYPE = "21686148-6449-6e6f-744e-656564454649"
LDM_PARTTYPES = {
"5808c8aa-7e8f-42e0-85d2-e1e90434cfb3", # LDM metadata (dynamic disk, GPT)
"af9b60a0-1431-4f62-bc68-3311714a69ad", # LDM data (dynamic disk, GPT)
"0x42", # dynamic disk, MBR
}
@dataclass
class BlockDev:
name: str
path: str
type: str
fstype: str | None = None
uuid: str | None = None
partuuid: str | None = None
label: str | None = None
parttype: str | None = None
pttype: str | None = None
size: int = 0
removable: bool = False
disk: str | None = None # name of the top-level disk
parent: str | None = None # name of the direct parent
def partition_number(self) -> int | None:
if self.type != "part" or not self.parent:
return None
match = re.fullmatch(re.escape(self.parent) + r"p?(\d+)", self.name)
return int(match.group(1)) if match else None
@property
def is_medium(self) -> bool:
return self.type == "disk" and self.fstype in MEDIA_FSTYPES
@property
def is_esp(self) -> bool:
return (self.parttype or "").lower() in ESP_PARTTYPES
def _flag(value) -> bool:
return value in (True, 1, "1", "true")
def parse_lsblk(text: str) -> list[BlockDev]:
data = json.loads(text)
devices: dict[str, BlockDev] = {}
def walk(node: dict, disk: str | None, parent: str | None) -> None:
dev = BlockDev(
name=node["name"],
path=node.get("path") or f"/dev/{node['name']}",
type=node.get("type") or "",
fstype=node.get("fstype"),
uuid=node.get("uuid"),
partuuid=node.get("partuuid"),
label=node.get("label"),
parttype=node.get("parttype"),
pttype=node.get("pttype"),
size=int(node.get("size") or 0),
removable=_flag(node.get("rm")),
parent=parent,
)
dev.disk = disk or (dev.name if dev.type == "disk" else None)
# a device can appear under several parents (RAID, multi-PV LVM)
devices.setdefault(dev.path, dev)
for child in node.get("children") or []:
walk(child, dev.disk, dev.name)
for node in data.get("blockdevices", []):
walk(node, None, None)
return list(devices.values())
def list_block_devices() -> list[BlockDev]:
proc = run(["lsblk", "-J", "-b", "-o", LSBLK_COLUMNS])
return parse_lsblk(proc.stdout)
def guest_disks(devices: list[BlockDev]) -> list[BlockDev]:
"""Guest disks in device order (xvda, xvdb, ..., xvdz, xvdaa)."""
disks = [
d for d in devices
if d.type == "disk" and not d.removable and not d.is_medium
and not d.name.startswith(IGNORED_DISK_PREFIXES)
]
return sorted(disks, key=lambda d: (len(d.name), d.name))
def activate_storage(readonly: bool) -> list[str]:
"""Assemble MD arrays and activate LVM volumes so their filesystems are visible.
Best effort: a guest without RAID or LVM makes these commands fail too. Returns the failures, which
tell why no OS was found when none is.
"""
mdadm = ["mdadm", "--assemble", "--scan"] + (["--readonly"] if readonly else [])
failures = []
for cmd in (mdadm, ["vgchange", "-ay"], ["udevadm", "settle", "--timeout=30"]):
try:
run(cmd, timeout=120)
except DockingError as exc:
failures.append(str(exc))
return failures

View File

@@ -1,90 +0,0 @@
"""Find the operating system(s) installed on the guest disks.
Every candidate filesystem is mounted read-only (without journal replay),
inspected, then unmounted.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from .blockdev import LDM_PARTTYPES, BlockDev
from .mounts import Mounts
from .report import Report
from .util import DockingError, find_ci, guest_exists
LINUX_FSTYPES = {"ext2", "ext3", "ext4", "xfs", "btrfs"}
# usual root subvolume names when the btrfs default subvolume is not the root
BTRFS_ROOT_SUBVOLS = ("@", "root", "@rootfs", "rootfs")
@dataclass
class OsCandidate:
kind: str # linux | windows
device: BlockDev
subvol: str | None = None
def classify(mnt: Path, fstype: str) -> str | None:
if fstype == "ntfs":
if find_ci(mnt, "Windows", "System32", "config", "SYSTEM"):
return "windows"
return None
if not (mnt / "etc" / "fstab").is_file():
return None
if guest_exists(mnt, "/etc/os-release") or guest_exists(mnt, "/usr/lib/os-release"):
return "linux"
if any((mnt / "etc").glob("*-release")):
return "linux"
return None
def probe(devices: list[BlockDev], mounts: Mounts, report: Report) -> list[OsCandidate]:
candidates: list[OsCandidate] = []
for dev in devices:
if (dev.parttype or "").lower() in LDM_PARTTYPES:
report.error("windows_dynamic_disk",
f"{dev.path} belongs to a Windows dynamic disk, which is not supported",
device=dev.path)
continue
if dev.fstype == "crypto_LUKS":
report.warning("luks_volume", f"{dev.path} is LUKS-encrypted and cannot be inspected",
device=dev.path)
continue
if dev.fstype == "BitLocker":
report.warning("bitlocker_volume",
f"{dev.path} is BitLocker-encrypted and cannot be inspected",
device=dev.path)
continue
if dev.fstype not in LINUX_FSTYPES | {"ntfs"}:
continue
try:
mnt = mounts.mount(dev.path, dev.fstype, readonly=True)
except DockingError as exc:
if exc.code == "windows_hibernated":
report.error(exc.code, str(exc), device=dev.path)
else:
report.warning("probe_mount_failed", str(exc), device=dev.path)
continue
try:
kind = classify(mnt, dev.fstype)
subvol = None
if kind is None and dev.fstype == "btrfs":
kind, subvol = _probe_btrfs_subvols(dev, mounts)
if kind:
candidates.append(OsCandidate(kind, dev, subvol))
finally:
mounts.umount(mnt)
return candidates
def _probe_btrfs_subvols(dev: BlockDev, mounts: Mounts) -> tuple[str | None, str | None]:
top = mounts.mount(dev.path, "btrfs", readonly=True, options=["subvolid=5"])
try:
for name in BTRFS_ROOT_SUBVOLS:
if (top / name).is_dir() and classify(top / name, "btrfs") == "linux":
return "linux", name
finally:
mounts.umount(top)
return None, None

View File

@@ -1,114 +0,0 @@
"""Firmware checks and UEFI boot path.
The VM's NVRAM is new on XCP-ng, so the boot entries created on VMware are
gone. Instead of writing NVRAM, make sure the removable-media path
\\EFI\\BOOT\\BOOTX64.EFI exists on the ESP: the firmware always tries it, and
shim's fallback (RHEL, Ubuntu...) recreates proper entries on first boot.
This is what `grub-install --force-extra-removable` does.
"""
from __future__ import annotations
import shutil
from pathlib import Path
from .blockdev import BIOS_BOOT_PARTTYPE, BlockDev
from .mounts import Mounts
from .report import Report
from .util import DockingError, find_ci
SKIPPED_VENDOR_DIRS = {"boot", "microsoft"}
# os-release ID -> directory name used on the ESP
VENDOR_ALIASES = {
"rhel": "redhat",
"ol": "redhat",
"opensuse-leap": "opensuse",
"opensuse-tumbleweed": "opensuse",
}
def firmware() -> str:
return "uefi" if Path("/sys/firmware/efi").exists() else "bios"
def find_esp(devices: list[BlockDev], disk: str | None) -> BlockDev | None:
esps = [d for d in devices if d.type == "part" and d.is_esp]
return next((d for d in esps if d.disk == disk), esps[0] if esps else None)
def fallback_copies(efi_dir: Path, kind: str, os_ids: list[str]) -> dict[Path, str] | None:
"""Files to copy into EFI/BOOT (source -> target name), {} if nothing to do."""
boot = find_ci(efi_dir, "BOOT")
if boot is not None and find_ci(boot, "BOOTX64.EFI") is not None:
return {}
if kind == "windows":
source = find_ci(efi_dir, "Microsoft", "Boot", "bootmgfw.efi")
return {source: "BOOTX64.EFI"} if source else None
vendors = [d for d in efi_dir.iterdir() if d.is_dir() and d.name.lower() not in SKIPPED_VENDOR_DIRS]
vendors = [d for d in vendors if find_ci(d, "shimx64.efi") or find_ci(d, "grubx64.efi")]
if not vendors:
return None
wanted = []
for os_id in (i.lower() for i in os_ids if i):
wanted += [os_id, VENDOR_ALIASES.get(os_id, os_id)]
vendors.sort(key=lambda d: wanted.index(d.name.lower()) if d.name.lower() in wanted else len(wanted))
vendor = vendors[0]
copies: dict[Path, str] = {}
shim = find_ci(vendor, "shimx64.efi")
grub = find_ci(vendor, "grubx64.efi")
if shim:
copies[shim] = "BOOTX64.EFI"
if grub:
copies[grub] = "grubx64.efi"
mok = find_ci(vendor, "mmx64.efi")
if mok:
copies[mok] = "mmx64.efi"
else:
copies[grub] = "BOOTX64.EFI"
config = find_ci(vendor, "grub.cfg")
if config:
copies[config] = "grub.cfg"
return copies
def plan(report: Report, mounts: Mounts, devices: list[BlockDev], disk: BlockDev,
kind: str, os_ids: list[str], readonly: bool) -> None:
mode = firmware()
report.firmware = mode
if mode == "bios":
if disk.pttype == "gpt":
if kind == "windows":
report.error("firmware_mismatch", "Windows on a GPT disk needs UEFI firmware on the VM")
elif not any(d.disk == disk.name and (d.parttype or "").lower() == BIOS_BOOT_PARTTYPE
for d in devices):
report.warning("firmware_mismatch",
"GPT disk without a BIOS boot partition: the VM probably needs UEFI firmware")
return
esp = find_esp(devices, disk.name)
if esp is None:
report.error("firmware_mismatch", "UEFI firmware but no EFI system partition: the VM probably needs BIOS")
return
try:
mnt = mounts.mounted(esp.path) or mounts.mount(esp.path, "vfat", readonly=readonly)
except DockingError as exc:
report.error("esp_mount_failed", str(exc), device=esp.path)
return
efi_dir = find_ci(mnt, "EFI")
copies = fallback_copies(efi_dir, kind, os_ids) if efi_dir else None
if copies is None:
report.error("no_efi_loader", f"no boot loader found on the ESP {esp.path}", device=esp.path)
return
if not copies:
report.info("efi_fallback_present", "the ESP already has \\EFI\\BOOT\\BOOTX64.EFI")
return
def apply() -> None:
boot = find_ci(efi_dir, "BOOT") or (efi_dir / "BOOT")
boot.mkdir(exist_ok=True)
for source, name in copies.items():
shutil.copyfile(source, boot / name)
names = ", ".join(sorted(f"{s.name}->{n}" for s, n in copies.items()))
report.plan("efi_fallback_loader", f"create \\EFI\\BOOT on {esp.path} ({names})", apply)

View File

@@ -1,101 +0,0 @@
"""Small wrapper over python3-hivex (offline Windows registry hives)."""
from __future__ import annotations
import gc
import struct
from pathlib import Path
REG_SZ = 1
REG_EXPAND_SZ = 2
REG_DWORD = 4
REG_MULTI_SZ = 7
def _utf16(text: str) -> bytes:
return (text + "\x00").encode("utf-16-le")
class Hive:
def __init__(self, path: Path, write: bool = False):
import hivex # Debian package python3-hivex, imported lazily for unit tests
self.path = Path(path)
self._h = hivex.Hivex(str(path), write=write)
def close(self) -> None:
# hivex has no explicit close; the handle (and its mmap) goes with the object
if self._h is not None:
self._h = None
gc.collect()
# -- navigation --
def root(self) -> int:
return self._h.root()
def child(self, node: int, name: str) -> int | None:
return self._h.node_get_child(node, name) # case-insensitive, None if missing
def node(self, *path: str) -> int | None:
current = self.root()
for name in path:
current = self.child(current, name)
if current is None:
return None
return current
def children(self, node: int) -> list[tuple[str, int]]:
return [(self._h.node_name(c), c) for c in self._h.node_children(node)]
def add_child(self, node: int, name: str) -> int:
existing = self.child(node, name)
return existing if existing is not None else self._h.node_add_child(node, name)
def delete_child(self, node: int) -> None:
self._h.node_delete_child(node)
# -- values --
def _value(self, node: int, key: str) -> int | None:
wanted = key.lower()
for value in self._h.node_values(node):
if self._h.value_key(value).lower() == wanted:
return value
return None
def dword(self, node: int, key: str) -> int | None:
value = self._value(node, key)
if value is None or self._h.value_type(value)[0] != REG_DWORD:
return None
return self._h.value_dword(value)
def string(self, node: int, key: str) -> str | None:
value = self._value(node, key)
if value is None or self._h.value_type(value)[0] not in (REG_SZ, REG_EXPAND_SZ):
return None
return self._h.value_string(value)
def strings(self, node: int, key: str) -> list[str]:
value = self._value(node, key)
if value is None:
return []
kind = self._h.value_type(value)[0]
if kind == REG_MULTI_SZ:
return [s for s in self._h.value_multiple_strings(value) if s]
if kind in (REG_SZ, REG_EXPAND_SZ):
text = self._h.value_string(value)
return [text] if text else []
return []
def set_dword(self, node: int, key: str, number: int) -> None:
self._h.node_set_value(node, {"key": key, "t": REG_DWORD, "value": struct.pack("<I", number)})
def set_string(self, node: int, key: str, text: str, *, expand: bool = False) -> None:
kind = REG_EXPAND_SZ if expand else REG_SZ
self._h.node_set_value(node, {"key": key, "t": kind, "value": _utf16(text)})
def set_strings(self, node: int, key: str, texts: list[str]) -> None:
data = "".join(t + "\x00" for t in texts) + "\x00"
self._h.node_set_value(node, {"key": key, "t": REG_MULTI_SZ, "value": data.encode("utf-16-le")})
def commit(self) -> None:
self._h.commit(None)

View File

@@ -1,938 +0,0 @@
"""Linux guests: inspection and docking planning.
Everything that touches the guest runs the guest's own tools through chroot
(dracut, update-initramfs, mkinitcpio); the live system's Python only reads
and edits files.
"""
from __future__ import annotations
import contextlib
import fnmatch
import logging
import os
import re
import shutil
import tarfile
from dataclasses import dataclass
from pathlib import Path
from .blockdev import BlockDev
from .detect import OsCandidate
from .mounts import Mounts
from .report import Report
from .util import DockingError, chroot_run, find_ci, guest_exists, guest_read, resolve_in_root, run
log = logging.getLogger("xo-docking")
# xen-blkfront is critical: a PVHVM kernel unplugs the emulated disks at boot
# whenever it is built with blkfront support, even as a module. If the module
# is not in the initramfs, the root disk disappears.
XEN_MODULES = ("xen-blkfront", "xen-netfront")
# fallback for kernels without Xen support: QEMU's emulated IDE controller
EMULATED_MODULES = ("ata_piix", "ata_generic", "sd_mod")
WANTED_MODULES = XEN_MODULES + EMULATED_MODULES
VMWARE_UNITS = (
"vmtoolsd.service",
"open-vm-tools.service",
"vgauth.service",
"vgauthd.service", # SUSE and RHEL name of vgauth.service
"vmware-tools.service",
"vmware-tools-thinprint.service",
)
UNIT_DIRS = ("/etc/systemd/system", "/usr/lib/systemd/system", "/lib/systemd/system")
SKIPPED_FSTYPES = {
"swap", "nfs", "nfs4", "cifs", "smb3", "smbfs", "tmpfs", "proc", "sysfs", "devpts",
"devtmpfs", "none", "iso9660", "udf", "fuse", "glusterfs", "ceph", "9p", "virtiofs",
}
ESSENTIAL_MOUNTPOINTS = {"/boot", "/usr"}
# by-id names derived from the content (LVM, MD, device-mapper UUIDs) survive the
# migration; the hardware ones (scsi-, wwn-, ata-...) do not
STABLE_BY_ID_PREFIXES = ("dm-uuid-", "dm-name-", "lvm-pv-uuid-", "md-uuid-", "md-name-")
UNSTABLE_SPEC_RE = re.compile(
r"^/dev/(sd[a-z]+\d*|hd[a-z]+\d*|disk/by-path/.+|disk/by-id/(?!" + "|".join(STABLE_BY_ID_PREFIXES) + r").+)$")
SD_SPEC_RE = re.compile(r"^/dev/(?:sd|hd)([a-z]+)(\d*)$")
BY_NAME_SPEC_RE = re.compile(r"^/dev/disk/(by-id|by-path)/(.+?)(?:-part(\d+))?$")
UNSTABLE_RESUME_RE = re.compile(r"\bresume=(/dev/(?:sd|hd)[a-z]+\d*|/dev/disk/by-(?:id|path)/[^\s\"']+)")
# kdump initrds built on VMware lack xen-blkfront and the guest does not rebuild
# them after the migration: the capture kernel then cannot find the disk.
# Deleting them is enough: SUSE's kdump.service (mkdumprd) and the RHEL
# family's kdumpctl regenerate a missing initrd when kdump starts.
SUSE_KDUMP_INITRD = "/var/lib/kdump/initrd"
RHEL_KDUMP_IMAGE_GLOB = "initramfs-*kdump.img"
IFACE_RE = re.compile(r"\b(eth\d+|ens\d+|eno\d+|enp\d+s\d+(?:f\d+)?)\b")
# kernel names: unchanged on XCP-ng (net.ifnames=0), and reserved for the kernel
KERNEL_IFACE_RE = re.compile(r"^eth\d+$")
MAC_RE = re.compile(r"^[0-9a-f]{2}(:[0-9a-f]{2}){5}$")
UDEVD_PATHS = ("/usr/lib/systemd/systemd-udevd", "/lib/systemd/systemd-udevd")
# xe-guest-utilities (XCP-ng Linux guest tools) or the newer xen-guest-agent
AGENT_PATHS = ("/usr/sbin/xe-daemon", "/usr/bin/xe-daemon",
"/usr/sbin/xen-guest-agent", "/usr/bin/xen-guest-agent")
AGENT_SERVICE = "xe-linux-distribution"
SYSTEMD_PATHS = ("/usr/lib/systemd/systemd", "/lib/systemd/systemd")
DEB_ARCHES = {"x86_64": "amd64", "i386": "i386"}
TGZ_ARCHES = {"x86_64": "x86_64", "i386": "386"}
# systemd unit shipped next to the packages, for the generic tgz (which only has an init script)
AGENT_UNIT_FILE = f"{AGENT_SERVICE}.service"
# ELF e_machine -> architecture name used by the guest tools packages
ELF_MACHINES = {0x3E: "x86_64", 0x03: "i386"}
POLICY_RC_D = "/usr/sbin/policy-rc.d"
TRANSACTIONAL_UPDATE = "/usr/sbin/transactional-update"
STAGING_DIR = "/tmp/xo-docking"
MARKER = "# added by xo-docking"
# -- pure helpers (unit tested) --------------------------------------------
@dataclass
class FstabEntry:
lineno: int
spec: str
mountpoint: str
fstype: str
options: str
def parse_os_release(text: str) -> dict:
values = {}
for line in text.splitlines():
key, sep, value = line.strip().partition("=")
if sep and key and not key.startswith("#"):
values[key] = value.strip().strip('"').strip("'")
return values
def parse_fstab(text: str) -> list[FstabEntry]:
entries = []
for lineno, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
fields = stripped.split()
if len(fields) < 3:
continue
options = fields[3] if len(fields) > 3 else "defaults"
entries.append(FstabEntry(lineno, fields[0], fields[1], fields[2], options))
return entries
def mountable_entries(entries: list[FstabEntry]) -> list[FstabEntry]:
"""Entries to mount under the guest root, parents first.
`noauto` entries are skipped: they are mounted on demand (CD drives,
removable disks, network shares) and may designate the docking CD itself.
The fstype field can be a list ("udf,iso9660").
"""
mountable = []
for e in entries:
options = e.options.split(",")
if not e.mountpoint.startswith("/") or e.mountpoint == "/":
continue
if "noauto" in options or "bind" in options:
continue
if any(t in SKIPPED_FSTYPES for t in e.fstype.split(",")):
continue
mountable.append(e)
return sorted(mountable, key=lambda e: e.mountpoint.count("/"))
def is_unstable_spec(spec: str) -> bool:
return bool(UNSTABLE_SPEC_RE.match(spec))
def _letters_to_index(letters: str) -> int:
index = 0
for char in letters:
index = index * 26 + (ord(char) - ord("a") + 1)
return index - 1
def map_sd_spec(spec: str, disks: list[BlockDev], devices: list[BlockDev]) -> BlockDev | None:
"""Map /dev/sdXN (VMware naming) to the device it designates now.
Assumes XO kept the disk order: the n-th VMware disk is the n-th disk here.
"""
match = SD_SPEC_RE.match(spec)
if not match:
return None
index = _letters_to_index(match.group(1))
if index >= len(disks):
return None
disk = disks[index]
if not match.group(2):
return disk
number = int(match.group(2))
return next(
(d for d in devices
if d.type == "part" and d.parent == disk.name and d.partition_number() == number),
None,
)
def map_by_name_spec(spec: str, anchor_spec: str | None, anchor: BlockDev | None,
devices: list[BlockDev]) -> BlockDev | None:
"""Map a /dev/disk/by-id or by-path name relative to a known entry (the root fs).
VMware ids and paths do not exist on XCP-ng, but the partitions of a disk
share its base name (`<base>-partN`): once the device of the anchor is known,
the other partitions of the same disk follow. Other disks stay unresolved.
"""
match = BY_NAME_SPEC_RE.match(spec)
anchor_match = BY_NAME_SPEC_RE.match(anchor_spec or "")
if anchor is None or not match or not anchor_match or not anchor_match.group(3):
return None
if match.group(1, 2) != anchor_match.group(1, 2):
return None
if anchor.partition_number() != int(anchor_match.group(3)):
return None # the anchor is not the partition its name says
disk = anchor.parent
if not match.group(3):
return next((d for d in devices if d.name == disk), None)
number = int(match.group(3))
return next(
(d for d in devices if d.type == "part" and d.parent == disk and d.partition_number() == number),
None,
)
def unstable_resume(text: str) -> str | None:
"""resume= device of /etc/default/grub that will not exist on XCP-ng."""
match = UNSTABLE_RESUME_RE.search(text)
return match.group(1) if match else None
def stale_kdump_initrds(root: Path) -> list[str]:
"""kdump initrd of SUSE, rebuilt by kdump.service once deleted."""
return [SUSE_KDUMP_INITRD] if resolve_in_root(root, SUSE_KDUMP_INITRD).is_file() else []
def rhel_kdump_images(root: Path) -> list[str]:
boot = resolve_in_root(root, "/boot")
return sorted(f"/boot/{p.name}" for p in boot.glob(RHEL_KDUMP_IMAGE_GLOB)) if boot.is_dir() else []
def rewrite_fstab(text: str, replacements: dict[int, str]) -> str:
"""Replace the first field of the given lines, keeping the rest untouched."""
lines = text.splitlines(keepends=True)
for lineno, spec in replacements.items():
match = re.match(r"^(\s*)(\S+)(.*)$", lines[lineno - 1], re.S)
lines[lineno - 1] = match.group(1) + spec + match.group(3)
return "".join(lines)
def detect_initramfs_tool(root: Path) -> str | None:
tools = (
("initramfs-tools", ("/usr/sbin/update-initramfs", "/sbin/update-initramfs")),
("dracut", ("/usr/bin/dracut", "/usr/sbin/dracut", "/sbin/dracut")),
("mkinitcpio", ("/usr/bin/mkinitcpio",)),
)
for name, paths in tools:
if any(guest_exists(root, p) for p in paths):
return name
return None
def kernel_versions(root: Path) -> list[str]:
moddir = resolve_in_root(root, "/lib/modules")
if not moddir.is_dir():
return []
boot = resolve_in_root(root, "/boot")
versions = []
for entry in sorted(moddir.iterdir()):
if not (entry / "modules.dep").is_file():
continue
if (boot / f"vmlinuz-{entry.name}").exists() or (entry / "vmlinuz").exists():
versions.append(entry.name)
return versions
def _canonical(module: str) -> str:
return module.replace("-", "_")
def available_modules(root: Path, kver: str, wanted=WANTED_MODULES) -> list[str]:
"""Modules from `wanted` that exist (loadable or built-in) for this kernel."""
moddir = resolve_in_root(root, f"/lib/modules/{kver}")
names = set()
for filename in ("modules.dep", "modules.builtin"):
path = moddir / filename
if not path.is_file():
continue
for line in path.read_text(errors="replace").splitlines():
base = line.split(":", 1)[0].strip().rsplit("/", 1)[-1]
base = re.sub(r"\.ko(\.(xz|gz|zst))?$", "", base)
if base:
names.add(_canonical(base))
return [m for m in wanted if _canonical(m) in names]
def readonly_root(root: Path, fstype: str | None, entries: list[FstabEntry]) -> bool:
"""Root of a transactional system (SL Micro...): a read-only btrfs snapshot.
Such a root is only changed through transactional-update, never offline.
"""
if fstype == "btrfs":
try:
proc = run(["btrfs", "property", "get", "-ts", str(root), "ro"], check=False)
if proc.stdout.strip() == "ro=true":
return True
except DockingError:
pass # no btrfs-progs: fall back to fstab
root_entry = next((e for e in entries if e.mountpoint == "/"), None)
return (root_entry is not None and "ro" in root_entry.options.split(",")
and guest_exists(root, TRANSACTIONAL_UPDATE))
def initramfs_files(image: Path) -> list[str]:
"""Paths inside an initramfs (early cpio and compressed main archive), [] if unreadable."""
try:
proc = run(["lsinitramfs", str(image)], check=False, timeout=300)
except DockingError:
return []
return proc.stdout.splitlines() if proc.returncode == 0 else []
def missing_initramfs_modules(files: list[str], modules) -> list[str]:
"""`modules` without a .ko file in the initramfs listing."""
present = {_canonical(re.sub(r"\.ko(\.(xz|gz|zst))?$", "", f.rsplit("/", 1)[-1]))
for f in files if re.search(r"\.ko(\.(xz|gz|zst))?$", f)}
return [m for m in modules if _canonical(m) not in present]
def initramfs_image(root: Path, kver: str) -> str | None:
boot = resolve_in_root(root, "/boot")
for name in (f"initramfs-{kver}.img", f"initrd.img-{kver}", f"initrd-{kver}"):
if (boot / name).exists():
return f"/boot/{name}"
return None
def interface_names(root: Path) -> list[str]:
"""Interface names referenced by the network configuration."""
patterns = (
"/etc/network/interfaces", "/etc/network/interfaces.d/*", "/etc/netplan/*.yaml",
"/etc/NetworkManager/system-connections/*", "/etc/sysconfig/network-scripts/ifcfg-*",
"/etc/sysconfig/network/ifcfg-*", "/etc/systemd/network/*.network",
)
names = set()
for pattern in patterns:
directory, _, glob = pattern.rpartition("/")
base = resolve_in_root(root, directory)
if not base.is_dir():
continue
for path in base.glob(glob):
if not path.is_file():
continue
names.update(IFACE_RE.findall(path.name))
try:
names.update(IFACE_RE.findall(path.read_text(errors="replace")))
except OSError:
continue
return sorted(names)
def parse_nic_map(value: str | None) -> dict[str, str]:
""""ens192=00:0c:29:aa:bb:cc;ens224=..." -> {name: mac}, invalid entries dropped."""
result = {}
for item in (value or "").split(";"):
name, sep, mac = item.strip().partition("=")
mac = mac.strip().lower().replace("-", ":")
if sep and name.strip() and MAC_RE.match(mac):
result[name.strip()] = mac
return result
def live_nics(sys_net: Path = Path("/sys/class/net")) -> dict[str, tuple[str, str]]:
"""Physical NICs of the running VM: {name: (mac, driver)}.
The VIFs keep the MACs of the VMware NICs, so they identify the old names.
"""
nics = {}
if not sys_net.is_dir():
return nics
for path in sorted(sys_net.iterdir()):
driver = path / "device" / "driver"
if not driver.exists():
continue # lo, bridges, bonds, vlans
try:
mac = (path / "address").read_text().strip().lower()
except OSError:
continue
nics[path.name] = (mac, os.path.basename(os.readlink(driver)))
return nics
def pin_interface_names(names: list[str], nics: dict[str, tuple[str, str]],
given: dict[str, str]) -> tuple[dict[str, tuple[str, str]], list[str]]:
"""Map the old interface names to (mac, driver), from XO's map or a single NIC.
Returns the pinned names and the names left unresolved.
"""
drivers = {mac: driver for mac, driver in nics.values()}
pinned, unresolved = {}, []
candidates = [n for n in names if not KERNEL_IFACE_RE.match(n)]
for name in candidates:
if name in given:
pinned[name] = (given[name], drivers.get(given[name], ""))
rest = [n for n in candidates if n not in pinned]
if len(rest) == 1 and len(nics) == 1 and not given:
pinned[rest[0]] = next(iter(nics.values()))
else:
unresolved = rest
return pinned, unresolved
def link_file(name: str, mac: str, driver: str) -> str:
"""systemd .link pinning `name` on the NIC with this MAC.
The driver match keeps VLANs (which inherit the MAC) out of it.
"""
match = f"MACAddress={mac}\n" + (f"Driver={driver}\n" if driver else "")
return f"{MARKER}\n[Match]\n{match}\n[Link]\nName={name}\n"
def disable_units(root: Path, units) -> list[str]:
"""Remove the enablement symlinks of `units`, like `systemctl disable`.
Done by hand because the live system's systemctl ignores /lib/systemd/system
(split-/usr guests) and may not understand an older guest's layout.
Masks (symlinks to /dev/null) are left alone.
"""
etc = resolve_in_root(root, "/etc/systemd/system")
if not etc.is_dir():
return []
removed = []
for link in sorted([*etc.glob("*.wants/*"), *etc.glob("*.requires/*"), *etc.glob("*")]):
if not link.is_symlink():
continue
target = os.readlink(link)
if target == "/dev/null":
continue
if link.name in units or Path(target).name in units:
link.unlink()
removed.append(str(link.relative_to(etc)))
return removed
def guest_arch(root: Path) -> str | None:
"""Architecture of the guest userland, from the ELF header of /bin/sh."""
try:
with resolve_in_root(root, "/bin/sh").open("rb") as handle:
header = handle.read(20)
except OSError:
return None
if len(header) < 20 or header[:4] != b"\x7fELF":
return None
order = "little" if header[5] == 1 else "big"
return ELF_MACHINES.get(int.from_bytes(header[18:20], order))
def package_manager(root: Path) -> str | None:
"""dpkg first: Debian guests can have the rpm tool installed too."""
if any(guest_exists(root, p) for p in ("/usr/bin/dpkg", "/bin/dpkg")):
return "dpkg"
if any(guest_exists(root, p) for p in ("/usr/bin/rpm", "/bin/rpm")):
return "rpm"
return None
def has_systemd(root: Path) -> bool:
return any(guest_exists(root, p) for p in SYSTEMD_PATHS)
def select_agent_package(names: list[str], manager: str, arch: str, systemd: bool) -> str | None:
"""xe-guest-utilities package of the guest tools ISO for this guest.
Same choice as the ISO's install.sh: the "legacy" rpm (chkconfig) is for
guests without systemd. "tgz" is the generic tarball, for systemd guests
with neither dpkg nor rpm (Arch...); it needs the unit file of the ISO.
"""
if manager == "dpkg":
if arch not in DEB_ARCHES:
return None
matches = [n for n in names if fnmatch.fnmatch(n, f"xe-guest-utilities_*_{DEB_ARCHES[arch]}.deb")]
elif manager == "rpm":
legacy = [n for n in names if fnmatch.fnmatch(n, f"xe-guest-utilities-*.legacy.{arch}.rpm")]
standard = [n for n in names
if fnmatch.fnmatch(n, f"xe-guest-utilities-*.{arch}.rpm") and n not in legacy]
matches = standard if systemd else legacy
elif manager == "tgz":
if not systemd or arch not in TGZ_ARCHES or AGENT_UNIT_FILE not in names:
return None
matches = [n for n in names if fnmatch.fnmatch(n, f"xe-guest-utilities_*_{TGZ_ARCHES[arch]}.tgz")]
else:
return None
return sorted(matches)[-1] if matches else None
def extract_agent_tgz(root: Path, tgz: Path) -> list[str]:
"""Extract the files and symlinks of the generic tarball under the guest root.
Paths are resolved like the guest does (/usr/sbin -> usr/bin on merged-/usr
guests). The SysV script is skipped: the systemd unit replaces it.
"""
installed = []
with tarfile.open(tgz) as archive:
for member in archive.getmembers():
name = member.name.removeprefix("./").lstrip("/")
if member.isdir() or name.startswith("etc/init.d/") or ".." in name.split("/"):
continue
target = resolve_in_root(root, "/" + name)
target.parent.mkdir(parents=True, exist_ok=True)
if target.is_symlink() or target.exists():
target.unlink()
if member.issym():
target.symlink_to(member.linkname)
elif member.isfile():
with archive.extractfile(member) as source, target.open("wb") as handle:
shutil.copyfileobj(source, handle)
target.chmod(member.mode & 0o7777)
else:
continue
installed.append("/" + name)
return installed
def enable_unit(root: Path, unit: str, target: str = "multi-user.target") -> None:
"""`systemctl enable` for a unit installed in /etc/systemd/system."""
wants = resolve_in_root(root, f"/etc/systemd/system/{target}.wants")
wants.mkdir(parents=True, exist_ok=True)
link = wants / unit
if link.is_symlink() or link.exists():
link.unlink()
link.symlink_to(f"/etc/systemd/system/{unit}")
def agent_enabled(root: Path) -> bool:
"""The agent service starts at boot (systemd unit or SysV rc links)."""
# the link itself: its absolute target does not resolve under `root`
wants = resolve_in_root(root, "/etc/systemd/system/multi-user.target.wants") / f"{AGENT_SERVICE}.service"
if wants.is_symlink():
return True
etc = resolve_in_root(root, "/etc")
return any(etc.glob(f"rc[2-5].d/S*{AGENT_SERVICE}")) or any(etc.glob(f"rc.d/rc[2-5].d/S*{AGENT_SERVICE}"))
@contextlib.contextmanager
def services_not_started(root: Path):
"""policy-rc.d forbidding invoke-rc.d to start services from the chroot.
In a chroot, invoke-rc.d does not see systemd and runs the init script:
the daemon would start on the live system and keep the guest mounted.
"""
policy = resolve_in_root(root, POLICY_RC_D)
backup = policy.with_name(policy.name + ".xo-docking.bak")
if policy.exists() or policy.is_symlink():
policy.rename(backup)
policy.write_text("#!/bin/sh\n" + MARKER + "\nexit 101\n")
policy.chmod(0o755)
try:
yield
finally:
policy.unlink()
if backup.exists() or backup.is_symlink():
backup.rename(policy)
def selinux_enabled(root: Path) -> bool:
text = guest_read(root, "/etc/selinux/config") or ""
match = re.search(r"^\s*SELINUX\s*=\s*(\w+)", text, re.M)
return bool(match) and match.group(1).lower() in ("enforcing", "permissive")
# -- guest -------------------------------------------------------------------
class LinuxGuest:
kind = "linux"
def __init__(self, candidate: OsCandidate, devices: list[BlockDev], disks: list[BlockDev],
mounts: Mounts, report: Report, readonly: bool, nic_map: dict[str, str] | None = None,
tools: Path | None = None):
self.nic_map = nic_map or {}
self.tools = tools # mounted guest tools disk
self.candidate = candidate
self.devices = devices
self.disks = disks
self.mounts = mounts
self.report = report
self.readonly = readonly
self.root: Path | None = None
self.os_ids: list[str] = []
self._root_spec: str | None = None # fstab spec of /, anchor for by-id/by-path names
self._chroot_ready = False
self.readonly_root = False # transactional system: nothing is written offline
self._not_planned: list[str] = [] # actions a read-only root prevents
# -- inspection --
def inspect(self) -> None:
dev = self.candidate.device
options = [f"subvol={self.candidate.subvol}"] if self.candidate.subvol else []
self.root = self.mounts.mount(dev.path, dev.fstype, readonly=self.readonly, options=options)
info = parse_os_release(
guest_read(self.root, "/etc/os-release") or guest_read(self.root, "/usr/lib/os-release") or ""
)
self.os_ids = [info.get("ID", "")] + info.get("ID_LIKE", "").split()
self.report.os = {
"kind": "linux",
"name": info.get("PRETTY_NAME") or info.get("NAME") or "unknown Linux",
"id": info.get("ID"),
"version": info.get("VERSION_ID"),
"root": dev.path,
}
fstab_text = guest_read(self.root, "/etc/fstab") or ""
entries = parse_fstab(fstab_text)
self._root_spec = next((e.spec for e in entries if e.mountpoint == "/"), None)
self.readonly_root = readonly_root(self.root, dev.fstype, entries)
if self.readonly_root:
self.report.os["readonly_root"] = True
self._mount_fstab(entries)
self._plan_fstab(fstab_text, entries)
self._plan_vmware_tools()
self._plan_network() # before the initramfs: the .link files are copied into it
self._plan_initramfs()
self._plan_kdump()
self._check_grub_defaults()
self._plan_guest_agent()
if self._not_planned:
self.report.warning("readonly_root",
"read-only root (transactional system), nothing is changed offline; "
"not done: " + ", ".join(self._not_planned)
+ ". Do it after the first boot with transactional-update",
actions=self._not_planned)
def finalize(self) -> None:
"""Planned last, once every other component has planned its actions."""
if self.report.actions and selinux_enabled(self.root):
self._plan("selinux_relabel",
"schedule a full SELinux relabel at next boot (one extra reboot)",
self._apply_selinux_relabel)
def _plan(self, code: str, description: str, apply, *, critical: bool = True) -> None:
"""Plan an action writing to the guest root, unless the root is read-only."""
if self.readonly_root:
self._not_planned.append(code)
return
self.report.plan(code, description, apply, critical=critical)
def close(self) -> None:
pass
def write_report(self, report: Report) -> None:
target = resolve_in_root(self.root, "/var/log/xo-docking")
target.mkdir(parents=True, exist_ok=True)
(target / "report.json").write_text(report.full_json())
log_file = Path("/run/xo-docking/xo-docking.log")
if log_file.exists():
shutil.copyfile(log_file, target / "xo-docking.log")
# -- fstab --
def _resolve_spec(self, spec: str) -> BlockDev | None:
for prefix, attr in (("UUID=", "uuid"), ("LABEL=", "label"), ("PARTUUID=", "partuuid")):
if spec.startswith(prefix):
wanted = spec[len(prefix):].strip('"').lower()
return next((d for d in self.devices if (getattr(d, attr) or "").lower() == wanted), None)
if spec.startswith("/dev/disk/by-uuid/"):
wanted = spec.rsplit("/", 1)[-1].lower()
return next((d for d in self.devices if (d.uuid or "").lower() == wanted), None)
mapped = self._map_unstable_spec(spec)
if mapped:
return mapped
if spec.startswith("/dev/"):
real = Path(spec).resolve()
return next((d for d in self.devices if Path(d.path).resolve() == real), None)
return None
def _mount_fstab(self, entries: list[FstabEntry]) -> None:
for entry in mountable_entries(entries):
essential = entry.mountpoint in ESSENTIAL_MOUNTPOINTS
dev = self._resolve_spec(entry.spec)
# a symlink of the guest must not lead the mount out of its root
target = resolve_in_root(self.root, entry.mountpoint)
if dev is None or not target.is_dir():
if essential:
self.report.error("fstab_mount_missing",
f"cannot find the device for {entry.mountpoint} ({entry.spec})",
mountpoint=entry.mountpoint)
continue
fstype = dev.fstype or entry.fstype
extra = [o for o in entry.options.split(",") if o.startswith(("subvol=", "subvolid="))]
try:
self.mounts.mount(dev.path, fstype, target, readonly=self.readonly, options=extra)
except DockingError as exc:
if essential:
self.report.error("fstab_mount_failed", str(exc), mountpoint=entry.mountpoint)
else:
self.report.warning("fstab_mount_failed", str(exc), mountpoint=entry.mountpoint)
def _plan_fstab(self, text: str, entries: list[FstabEntry]) -> None:
replacements: dict[int, str] = {}
unresolved = []
for entry in entries:
if not is_unstable_spec(entry.spec):
continue
dev = self._map_unstable_spec(entry.spec)
if dev is not None and dev.uuid:
replacements[entry.lineno] = f"UUID={dev.uuid}"
else:
unresolved.append(entry.spec)
if unresolved:
self.report.warning("fstab_unstable_names",
"fstab uses device names that may change on XCP-ng: " + ", ".join(unresolved),
specs=unresolved)
if not replacements:
return
new_text = rewrite_fstab(text, replacements)
def apply() -> None:
fstab = resolve_in_root(self.root, "/etc/fstab")
shutil.copy2(fstab, fstab.with_name("fstab.xo-docking.bak"))
# rewrite in place: keeps the inode, owner and SELinux label
with fstab.open("w") as handle:
handle.write(new_text)
self._plan("fstab_stable_names",
f"replace {len(replacements)} /dev/sdX or by-id/by-path reference(s) in fstab "
"with UUIDs", apply)
def _map_unstable_spec(self, spec: str) -> BlockDev | None:
return (map_sd_spec(spec, self.disks, self.devices)
or map_by_name_spec(spec, self._root_spec, self.candidate.device, self.devices))
# -- VMware tools --
def _plan_vmware_tools(self) -> None:
units = [u for u in VMWARE_UNITS
if any(guest_exists(self.root, f"{d}/{u}") for d in UNIT_DIRS)]
if not units:
return
def apply() -> None:
removed = disable_units(self.root, units)
log.info("removed unit links: %s", ", ".join(removed) or "none")
self._plan("disable_vmware_tools", "disable " + ", ".join(units), apply, critical=False)
# -- initramfs --
def _plan_initramfs(self) -> None:
tool = detect_initramfs_tool(self.root)
kernels = kernel_versions(self.root)
self.report.os["kernels"] = kernels
self.report.os["initramfs"] = tool
if not kernels:
self.report.error("no_kernel_found", "no installed kernel found in /lib/modules")
return
modules = {k: available_modules(self.root, k) for k in kernels}
without_blkfront = [k for k in kernels if "xen-blkfront" not in modules[k]]
if without_blkfront:
self.report.warning("no_xen_blkfront",
"kernel(s) without Xen PV disk driver, will boot on emulated IDE: "
+ ", ".join(without_blkfront), kernels=without_blkfront)
if self.readonly_root:
self._check_readonly_initramfs(kernels, modules)
return
if tool is None:
self.report.error("unsupported_initramfs_tool",
"no supported initramfs tool found (dracut, initramfs-tools, mkinitcpio)")
return
common = [m for m in WANTED_MODULES if all(m in modules[k] for k in kernels)]
if tool == "dracut":
def apply() -> None:
self._prepare_chroot()
for kver in kernels:
cmd = ["dracut", "--force", "--kver", kver]
if modules[kver]:
cmd += ["--add-drivers", " ".join(modules[kver])]
image = initramfs_image(self.root, kver)
if image:
cmd.append(image)
chroot_run(self.root, cmd)
elif tool == "initramfs-tools":
def apply() -> None:
self._append_module_list("/etc/initramfs-tools/modules", common)
self._prepare_chroot()
chroot_run(self.root, ["update-initramfs", "-u", "-k", "all"])
else:
self.report.warning("mkinitcpio_experimental", "mkinitcpio support is experimental")
def apply() -> None:
dropin = resolve_in_root(self.root, "/etc/mkinitcpio.conf.d")
dropin.mkdir(parents=True, exist_ok=True)
(dropin / "90-xo-docking.conf").write_text(f"{MARKER}\nMODULES+=({' '.join(common)})\n")
self._prepare_chroot()
chroot_run(self.root, ["mkinitcpio", "-P"])
self.report.plan("initramfs_xen_drivers",
f"rebuild the initramfs with {tool} including Xen and emulated disk drivers "
f"for {len(kernels)} kernel(s)", apply)
def _check_readonly_initramfs(self, kernels: list[str], modules: dict[str, list[str]]) -> None:
"""A read-only root cannot be rebuilt offline: check the existing initramfs instead.
Generic images (SL Micro...) ship an initramfs with every driver.
"""
missing = {}
for kver in kernels:
wanted = [m for m in XEN_MODULES if m in modules[kver]]
image = initramfs_image(self.root, kver)
files = initramfs_files(resolve_in_root(self.root, image)) if image else []
lacking = missing_initramfs_modules(files, wanted) if files else wanted
if lacking:
missing[kver] = lacking
if not missing:
self.report.info("initramfs_xen_drivers_present",
"the initramfs already has the Xen PV drivers for every kernel")
return
self.report.error("readonly_root_initramfs",
"read-only root: the initramfs of " + ", ".join(missing)
+ " lacks the Xen PV drivers and cannot be rebuilt offline. On the source, add "
'add_drivers+=" xen-blkfront xen-netfront " to /etc/dracut.conf.d/xen.conf, run '
"transactional-update initrd and reboot, then migrate again", kernels=missing)
# -- kdump --
def _plan_kdump(self) -> None:
stale = stale_kdump_initrds(self.root) + rhel_kdump_images(self.root)
if not stale:
return
def apply() -> None:
for path in stale:
resolve_in_root(self.root, path).unlink()
self._plan("kdump_initrd_rebuild",
"delete the kdump initrd built on VMware (" + ", ".join(stale)
+ "), kdump rebuilds it at the next boot", apply, critical=False)
def _append_module_list(self, path: str, modules: list[str]) -> None:
target = resolve_in_root(self.root, path)
current = target.read_text() if target.exists() else ""
present = {line.strip() for line in current.splitlines()}
missing = [m for m in modules if m not in present]
if not missing:
return
if target.exists():
shutil.copy2(target, target.with_name(target.name + ".xo-docking.bak"))
with target.open("a") as handle:
if current and not current.endswith("\n"):
handle.write("\n")
handle.write(MARKER + "\n" + "\n".join(missing) + "\n")
def _prepare_chroot(self) -> None:
if self._chroot_ready:
return
for source in ("/dev", "/dev/pts", "/proc", "/sys"):
target = self.root / source.lstrip("/")
if not target.is_dir():
raise DockingError("chroot_setup_failed", f"{source} is missing in the guest root")
self.mounts.bind(source, target)
self._chroot_ready = True
# -- network --
def _plan_network(self) -> None:
names = interface_names(self.root)
if not names:
return
if any(guest_exists(self.root, p) for p in UDEVD_PATHS):
pinned, unresolved = pin_interface_names(names, live_nics(), self.nic_map)
else:
pinned, unresolved = {}, [n for n in names if not KERNEL_IFACE_RE.match(n)]
if unresolved:
self.report.warning("linux_interface_names",
"network configuration references interface names that will change on "
"XCP-ng: " + ", ".join(unresolved), interfaces=unresolved)
if not pinned:
return
def apply() -> None:
directory = resolve_in_root(self.root, "/etc/systemd/network")
directory.mkdir(parents=True, exist_ok=True)
for name, (mac, driver) in pinned.items():
(directory / f"10-xo-docking-{name}.link").write_text(link_file(name, mac, driver))
self._plan("linux_pin_interface_names",
"keep the interface names " + ", ".join(f"{n} ({m})" for n, (m, _) in pinned.items())
+ " with systemd .link files", apply, critical=False)
# -- guest agent --
def _plan_guest_agent(self) -> None:
if any(guest_exists(self.root, p) for p in AGENT_PATHS):
return
if self.tools is None:
self.report.info("guest_agent_missing",
"xe-guest-utilities is not installed and no guest tools disk is attached; "
"install it after the first boot")
return
arch = guest_arch(self.root)
systemd = has_systemd(self.root)
manager = package_manager(self.root) or ("tgz" if systemd else None)
directory = find_ci(self.tools, "Linux")
names = sorted(p.name for p in directory.iterdir()) if directory else []
package = select_agent_package(names, manager, arch, systemd) if manager and arch else None
if package is None:
self.report.warning("guest_agent_unsupported",
"no xe-guest-utilities package on the guest tools disk for this guest "
f"({manager or 'neither dpkg, rpm nor systemd'}, {arch or 'unknown architecture'}); "
"install it after the first boot", manager=manager, arch=arch)
return
self.report.os["guest_agent"] = package
def apply() -> None:
self._install_agent(directory / package, manager, systemd)
self._plan("install_guest_agent", f"install {package} from the guest tools disk",
apply, critical=False)
def _install_agent(self, package: Path, manager: str, systemd: bool) -> None:
if manager == "tgz":
installed = extract_agent_tgz(self.root, package)
log.info("installed from %s: %s", package.name, ", ".join(installed))
unit = resolve_in_root(self.root, f"/etc/systemd/system/{AGENT_UNIT_FILE}")
shutil.copyfile(package.with_name(AGENT_UNIT_FILE), unit)
unit.chmod(0o644)
enable_unit(self.root, AGENT_UNIT_FILE)
return
self._prepare_chroot()
staging = resolve_in_root(self.root, STAGING_DIR)
staging.mkdir(parents=True, exist_ok=True)
shutil.copyfile(package, staging / package.name)
guest_path = f"{STAGING_DIR}/{package.name}"
try:
if manager == "dpkg":
with services_not_started(self.root):
chroot_run(self.root, ["dpkg", "-i", guest_path])
elif systemd:
# the scriptlets enable the unit; systemctl does not start anything in a chroot
chroot_run(self.root, ["rpm", "-U", "--replacepkgs", guest_path])
else:
# the legacy scriptlets would start the daemon on the live system
chroot_run(self.root, ["rpm", "-U", "--replacepkgs", "--noscripts", guest_path])
chroot_run(self.root, ["chkconfig", "--add", AGENT_SERVICE])
finally:
shutil.rmtree(staging, ignore_errors=True)
if not agent_enabled(self.root) and systemd:
chroot_run(self.root, ["systemctl", "enable", f"{AGENT_SERVICE}.service"])
if not agent_enabled(self.root):
raise DockingError("guest_agent_not_enabled", f"{AGENT_SERVICE} is installed but not enabled at boot")
# -- checks only --
def _check_grub_defaults(self) -> None:
text = guest_read(self.root, "/etc/default/grub") or ""
if re.search(r"root=/dev/(sd|hd)[a-z]", text):
self.report.warning("grub_unstable_root", "/etc/default/grub sets root= with a /dev/sdX name")
resume = unstable_resume(text)
if resume:
# the initramfs waits for this device at every boot
self.report.warning("grub_unstable_resume",
f"/etc/default/grub sets resume={resume}, which will not exist on XCP-ng",
device=resume)
def _apply_selinux_relabel(self) -> None:
resolve_in_root(self.root, "/.autorelabel").touch()

View File

@@ -1,151 +0,0 @@
"""Mount tracking: every mount is undone in reverse order on exit."""
from __future__ import annotations
import logging
import time
from pathlib import Path
from .util import DockingError, run
log = logging.getLogger("xo-docking")
# ntfs-3g exit codes worth explaining to the user
NTFS3G_ERRORS = {
14: ("windows_hibernated",
"Windows is hibernated or was stopped with Fast Startup; do a full shutdown on the source"),
15: ("ntfs_unclean",
"the NTFS volume was not cleanly unmounted; run chkdsk on the source"),
}
def mount_options(fstype: str, readonly: bool, extra=()) -> list[str]:
"""Options guaranteeing a read-only mount does not write (no journal replay)."""
if fstype == "ntfs":
options = ["ro"] if readonly else []
else:
options = ["ro" if readonly else "rw"]
if readonly:
if fstype in ("ext3", "ext4"):
options.append("noload")
elif fstype == "xfs":
options.append("norecovery")
elif fstype == "btrfs":
options.append("rescue=nologreplay")
options.extend(o for o in extra if o)
return options
def is_readonly_mount(target: Path, mounts_file: str = "/proc/self/mounts") -> bool:
"""Whether the filesystem mounted on `target` is read-only."""
try:
lines = Path(mounts_file).read_text().splitlines()
except OSError:
return False
wanted = str(Path(target).resolve())
for line in reversed(lines): # the last mount on a path hides the others
fields = line.split()
if len(fields) >= 4 and fields[1].replace("\\040", " ") == wanted:
return "ro" in fields[3].split(",")
return False
class Mounts:
def __init__(self, base: str | Path = "/run/xo-docking/mnt"):
self.base = Path(base)
self._stack: list[tuple[Path, bool]] = [] # (target, is_fuse)
self._devices: dict[Path, str] = {} # target -> device, bind mounts excluded
self._counter = 0
def __enter__(self) -> "Mounts":
return self
def __exit__(self, *exc) -> None:
self.close()
def new_dir(self, label: str = "m") -> Path:
self._counter += 1
path = self.base / f"{label}{self._counter}"
path.mkdir(parents=True, exist_ok=True)
return path
def mount(self, device: str, fstype: str, target: Path | None = None, *,
readonly: bool = True, options=()) -> Path:
target = Path(target) if target else self.new_dir()
opts = mount_options(fstype, readonly, options)
if fstype == "ntfs":
cmd = ["ntfs-3g"] + (["-o", ",".join(opts)] if opts else []) + [device, str(target)]
proc = run(cmd, check=False)
if proc.returncode != 0:
code, message = NTFS3G_ERRORS.get(
proc.returncode,
("ntfs_mount_failed", f"ntfs-3g exited with status {proc.returncode}"),
)
raise DockingError(code, f"{device}: {message}")
else:
proc = run(["mount", "-t", fstype, "-o", ",".join(opts), device, str(target)], check=False)
if proc.returncode != 0:
raise DockingError("mount_failed", f"cannot mount {device} ({fstype}) on {target}")
self._stack.append((target, fstype == "ntfs"))
self._devices[target] = device
if fstype == "ntfs" and not readonly and is_readonly_mount(target):
# ntfs-3g falls back to read-only, with success, on a volume left unclean by a power off:
# every later write would fail, after the hives were already changed in memory
self.umount(target)
raise DockingError("ntfs_unclean",
f"{device}: the NTFS volume was not cleanly shut down (powered off, or the VM was "
"imported while running), ntfs-3g only mounts it read-only. Shut Windows down "
"cleanly on the source, or boot it once, then import it again")
return target
def mounted(self, device: str) -> Path | None:
"""Where `device` is already mounted by us (e.g. the ESP under the guest's /boot/efi)."""
real = Path(device).resolve()
return next((t for t, d in self._devices.items() if Path(d).resolve() == real), None)
def bind(self, source: str | Path, target: Path) -> Path:
run(["mount", "--bind", str(source), str(target)])
self._stack.append((Path(target), False))
return Path(target)
def umount(self, target: Path) -> None:
for index in range(len(self._stack) - 1, -1, -1):
if self._stack[index][0] == Path(target):
entry = self._stack.pop(index)
self._devices.pop(entry[0], None)
self._umount(*entry)
return
def close(self) -> None:
while self._stack:
self._umount(*self._stack.pop())
self._devices.clear()
run(["sync"], check=False)
@staticmethod
def _umount(target: Path, fuse: bool) -> None:
if run(["umount", str(target)], check=False).returncode != 0:
log.warning("umount %s failed, detaching lazily", target)
run(["umount", "-l", str(target)], check=False)
if fuse:
_wait_fuse_exit(target)
def _wait_fuse_exit(target: Path, timeout: int = 120) -> None:
"""ntfs-3g flushes after umount returns: wait for its process to exit."""
deadline = time.monotonic() + timeout
needle = str(target).encode() + b"\x00" # cmdline arguments are NUL-terminated
while time.monotonic() < deadline:
busy = False
for cmdline in Path("/proc").glob("[0-9]*/cmdline"):
try:
content = cmdline.read_bytes()
except OSError:
continue
if b"ntfs-3g" in content and needle in content:
busy = True
break
if not busy:
return
time.sleep(0.5)
log.warning("ntfs-3g for %s still running after %ss", target, timeout)

View File

@@ -1,197 +0,0 @@
"""Report model: findings, planned actions and their outcome.
Detection never modifies anything: it records findings and *plans* actions.
In diagnose mode the planned actions are only reported; in dock mode they
are applied in order.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, field
from typing import Callable
log = logging.getLogger("xo-docking")
# oxenstored (XCP-ng) refuses guest values over quota-maxsize = 2048 bytes:
# the write fails and XO gets no report at all. Keep a safety margin.
XENSTORE_VALUE_LIMIT = 1900
# what XO reads in a report: the rest is only for humans, and dropped first when the report does not fit
XO_OS_KEYS = ("kind", "name", "rtc_timeoffset", "firstboot")
XO_NETWORK_KEYS = ("guid", "dhcp", "addresses")
XO_CODES = frozenset({
"boot_disk_not_first",
"guest_agent_missing",
"guest_agent_unsupported",
"install_firstboot",
"install_guest_agent",
"pv_drivers_unsupported",
"windows_firstboot_incomplete",
"windows_static_ip_unmapped",
})
@dataclass
class Finding:
level: str # info | warning | error
code: str
message: str
data: dict = field(default_factory=dict)
@dataclass
class Action:
code: str
description: str
apply: Callable[[], None] = field(repr=False)
critical: bool = True
status: str = "planned" # planned | applied | failed | skipped
error: str | None = None
class Report:
def __init__(self, mode: str, version: str):
self.mode = mode
self.version = version
self.firmware: str | None = None
self.disks: list[str] = []
self.tools: str | None = None # guest tools disk, see tools.py
self.os: dict = {}
# TCP/IP configuration of the old adapters (Windows), for XO to reapply
self.network: list[dict] = []
self.findings: list[Finding] = []
self.actions: list[Action] = []
# -- findings ---------------------------------------------------------
def _add(self, level: str, code: str, message: str, data: dict) -> None:
log.log(
{"info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}[level],
"[%s] %s", code, message,
)
self.findings.append(Finding(level, code, message, data))
def info(self, code: str, message: str, **data) -> None:
self._add("info", code, message, data)
def warning(self, code: str, message: str, **data) -> None:
self._add("warning", code, message, data)
def error(self, code: str, message: str, **data) -> None:
self._add("error", code, message, data)
@property
def blocked(self) -> bool:
return any(f.level == "error" for f in self.findings)
# -- actions ----------------------------------------------------------
def plan(self, code: str, description: str, apply: Callable[[], None], *, critical: bool = True) -> None:
log.info("planned: %s (%s)", code, description)
self.actions.append(Action(code, description, apply, critical))
def apply_actions(self) -> None:
for index, action in enumerate(self.actions):
log.info("applying: %s", action.code)
try:
action.apply()
action.status = "applied"
except Exception as exc: # noqa: BLE001 - every failure must be reported
log.exception("action %s failed", action.code)
action.status = "failed"
action.error = str(exc)
if action.critical:
self.error("action_failed", f"{action.code}: {exc}", action=action.code)
for remaining in self.actions[index + 1:]:
remaining.status = "skipped"
return
self.warning("action_failed", f"{action.code}: {exc}", action=action.code)
# -- output -----------------------------------------------------------
def result(self) -> str:
if self.blocked or any(a.status == "failed" and a.critical for a in self.actions):
return "error"
if any(f.level == "warning" for f in self.findings):
return "warning"
return "ok"
def to_dict(self, *, with_details: bool = True, with_data: bool = True, with_messages: bool = True,
with_info: bool = True) -> dict:
findings = []
for f in self.findings:
# an info XO acts on is kept
if f.level == "info" and not with_info and f.code not in XO_CODES:
continue
entry = {"level": f.level, "code": f.code}
if with_messages:
entry["message"] = f.message
if with_data and f.data:
entry["data"] = f.data
findings.append(entry)
actions = []
for a in self.actions:
entry = {"code": a.code, "status": a.status}
if with_messages:
entry["description"] = a.description
if a.error and with_messages:
entry["error"] = a.error
actions.append(entry)
return {
"version": self.version,
"mode": self.mode,
"result": self.result(),
"firmware": self.firmware,
"disks": self.disks,
"tools": self.tools,
"os": self.os if with_details else {k: self.os[k] for k in XO_OS_KEYS if k in self.os},
"network": self.network if with_details else [
{k: adapter[k] for k in XO_NETWORK_KEYS if k in adapter}
# XO only matches the static adapters with the cards of the source
for adapter in self.network if not adapter.get("dhcp") and adapter.get("addresses")
],
"findings": findings,
"actions": actions,
}
def full_json(self) -> str:
return json.dumps(self.to_dict(), indent=2)
def compact_json(self, limit: int = XENSTORE_VALUE_LIMIT) -> str:
"""Smallest faithful summary that fits in a xenstore value.
Detail is dropped progressively; codes are always kept because XO
relies on them, messages are only for humans.
"""
variants = (
{"with_details": True, "with_data": True, "with_messages": True, "with_info": True},
{"with_details": False, "with_data": True, "with_messages": True, "with_info": True},
{"with_details": False, "with_data": False, "with_messages": True, "with_info": True},
{"with_details": False, "with_data": False, "with_messages": True, "with_info": False},
{"with_details": False, "with_data": False, "with_messages": False, "with_info": False},
)
for options in variants:
text = json.dumps(self.to_dict(**options), separators=(",", ":"))
if len(text.encode()) <= limit:
return text
data = self.to_dict(with_details=False, with_data=False, with_messages=False, with_info=False)
data["truncated"] = True
while len(json.dumps(data, separators=(",", ":")).encode()) > limit:
longest = max(("findings", "actions"), key=lambda k: len(data[k]))
if not data[longest]:
break
_drop_one(data[longest])
return json.dumps(data, separators=(",", ":"))
def _is_relied_on(entry: dict) -> bool:
"""A finding or an action XO acts on, or which tells why the run failed."""
return entry.get("level") == "error" or entry.get("status") == "failed" or entry["code"] in XO_CODES
def _drop_one(entries: list[dict]) -> None:
"""Drops the last entry XO does not rely on, the last one when it relies on all of them."""
for index in range(len(entries) - 1, -1, -1):
if not _is_relied_on(entries[index]):
del entries[index]
return
entries.pop()

View File

@@ -1,33 +0,0 @@
"""Guest tools disk: the XCP-ng guest tools ISO, attached read-only by XO.
XO attaches the ISO from the pool's ISO SR as a read-only *disk*
(`userdevice` 25, xvdz), not as a CD: xen-blkfront ignores CD drives on HVM
guests, so the live system would not see it. It is told apart from the guest
disks by its filesystem, and recognized by its Linux/ or Windows/ directory.
"""
from __future__ import annotations
from pathlib import Path
from .blockdev import BlockDev
from .mounts import Mounts
from .report import Report
from .util import DockingError, find_ci
def find_tools(devices: list[BlockDev], mounts: Mounts, report: Report) -> Path | None:
"""Mount the guest tools disk read-only; None when XO did not attach one."""
for dev in devices:
if not dev.is_medium:
continue
try:
mnt = mounts.mount(dev.path, dev.fstype, readonly=True)
except DockingError as exc:
report.warning("tools_disk_mount_failed", str(exc), device=dev.path)
continue
if find_ci(mnt, "Linux") or find_ci(mnt, "Windows"):
report.tools = dev.name
return mnt
mounts.umount(mnt)
return None

View File

@@ -1,137 +0,0 @@
"""Shared helpers: errors, command execution and guest path resolution."""
from __future__ import annotations
import logging
import os
import shlex
import subprocess
from pathlib import Path
log = logging.getLogger("xo-docking")
# Environment used for every command run inside the guest (chroot).
CHROOT_ENV = {
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG": "C",
"LC_ALL": "C",
"HOME": "/root",
}
class DockingError(Exception):
"""Error carrying a stable, machine-readable code for the report."""
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = code
def _last_line(text: str) -> str:
lines = [line for line in text.strip().splitlines() if line.strip()]
return lines[-1] if lines else ""
def run(cmd, *, check: bool = True, timeout: int = 3600, env: dict | None = None):
"""Run a command, log its output at debug level, raise DockingError on failure."""
cmd = [str(c) for c in cmd]
log.info("$ %s", shlex.join(cmd))
try:
proc = subprocess.run(
cmd,
capture_output=True,
text=True,
errors="replace",
timeout=timeout,
env=env,
)
except FileNotFoundError as exc:
raise DockingError("missing_tool", f"{cmd[0]} is not available") from exc
except subprocess.TimeoutExpired as exc:
raise DockingError("command_timeout", f"{cmd[0]} timed out after {timeout}s") from exc
for stream in (proc.stdout, proc.stderr):
for line in stream.splitlines():
log.debug(" %s", line)
if check and proc.returncode != 0:
detail = _last_line(proc.stderr) or _last_line(proc.stdout)
raise DockingError(
"command_failed", f"{cmd[0]} exited with status {proc.returncode}: {detail}"
)
return proc
def chroot_run(root: Path, cmd, **kwargs):
"""Run a command with the guest's own binaries, inside its root."""
return run(["chroot", str(root), *cmd], env=CHROOT_ENV, **kwargs)
def resolve_in_root(root: Path, path: str) -> Path:
"""Resolve `path` the way the guest would, following symlinks inside `root`.
Absolute symlinks (e.g. /lib -> /usr/lib) must not escape to the live system.
"""
pending = [p for p in path.split("/") if p and p != "."]
resolved: list[str] = []
hops = 0
while pending:
part = pending.pop(0)
if part == "..":
if resolved:
resolved.pop()
continue
candidate = root.joinpath(*resolved, part)
if candidate.is_symlink():
hops += 1
if hops > 40:
raise DockingError("symlink_loop", f"too many symlinks while resolving {path}")
target = os.readlink(candidate)
if target.startswith("/"):
resolved = []
pending = [p for p in target.split("/") if p and p != "."] + pending
continue
resolved.append(part)
return root.joinpath(*resolved)
def guest_read(root: Path, path: str) -> str | None:
"""Read a text file from the guest, or None if it does not exist."""
target = resolve_in_root(root, path)
try:
return target.read_text(errors="replace")
except (FileNotFoundError, IsADirectoryError, NotADirectoryError, PermissionError):
return None
def guest_exists(root: Path, path: str) -> bool:
return resolve_in_root(root, path).exists()
def find_ci(base: Path | None, *parts: str) -> Path | None:
"""Case-insensitive path lookup (NTFS and FAT content seen from Linux)."""
current = base
for part in parts:
if current is None or not current.is_dir():
return None
exact = current / part
if exact.exists():
current = exact
continue
wanted = part.lower()
try:
current = next((c for c in current.iterdir() if c.name.lower() == wanted), None)
except OSError:
return None
return current
def kernel_cmdline_param(name: str, cmdline_path: str = "/proc/cmdline") -> str | None:
"""Return the value of `name=value` on the kernel command line ("" for a bare flag)."""
try:
args = Path(cmdline_path).read_text().split()
except OSError:
return None
for arg in args:
key, sep, value = arg.partition("=")
if key == name:
return value if sep else ""
return None

View File

@@ -1,614 +0,0 @@
"""Windows guests: inspection and docking planning.
Offline work covers what must happen before Windows can boot on XCP-ng, in
every control set (so "Last Known Good Configuration" stays bootable too):
- boot-start drivers for the emulated IDE controller;
- VMware services and drivers disabled (Start=4), found generically;
- static IP configuration of the old (VMware) adapters captured for XO, then
cleared, so Windows does not refuse the same address on the new adapter.
Anything requiring a running Windows (Xen PV drivers, VMware Tools removal,
optional deep cleanup) is done by a first-boot script, in phases.
"""
from __future__ import annotations
import fnmatch
import logging
import re
import shutil
from pathlib import Path
from .detect import OsCandidate
from .hive import Hive
from .linux import MAC_RE, live_nics
from .mounts import Mounts
from .report import Report
from .util import DockingError, find_ci
log = logging.getLogger("xo-docking")
MIN_BUILD = 7601 # Windows 7 SP1 / Server 2008 R2 SP1
# XCP-ng 9.x PV drivers: Windows 10 1607 / Server 2016 and later (their MSI
# also needs .NET 4.x and fails with 1603 on older systems)
PV9_MIN_BUILD = 14393
DEFAULT_MSI_PATTERNS = ("*.msi",)
# QEMU emulates a PIIX IDE controller: these drivers must start at boot (Start=0)
IDE_SERVICES = ("atapi", "intelide", "pciide")
# installed by the PV drivers MSI: their presence means a first boot already ran
PV_DRIVER_SERVICES = ("xenbus", "xenvbd")
# Services and drivers installed by VMware Tools. Others are found by
# is_vmware_service(); this list only guarantees the usual ones are covered.
KNOWN_VMWARE_SERVICES = (
# services
"VMTools", "VGAuthService", "vm3dservice", "vmvss",
"VMwareCAFCommAmqpListener", "VMwareCAFManagementAgentHost",
# drivers
"vmci", "vsock", "vmmemctl", "vmhgfs", "vmrawdsk", "vm3dmp", "vm3dmp_loader",
"vm3dmp-debug", "vm3dmp-stats", "vmmouse", "vmusbmouse", "vmxnet3ndis6", "vmxnet3",
"pvscsi", "vmkbd", "VMMemCtl",
)
TOOLS_PATH_MARKER = "\\vmware\\vmware tools\\"
SERVICE_KERNEL_DRIVER = 1
SERVICE_FS_DRIVER = 2
NET_CLASS = "{4d36e972-e325-11ce-bfc1-08002be10318}" # Control\Class\<network adapters>
GUID_RE = re.compile(r"^\{[0-9a-fA-F-]{36}\}$")
FIRSTBOOT_SERVICE = "XoDockingFirstboot"
FIRSTBOOT_DIR = "xo-docking" # at the root of the system volume
PACKAGES_DIR = "packages" # under FIRSTBOOT_DIR: MSIs copied from the guest tools disk
FIRSTBOOT_IMAGE_PATH = (
r'%SystemRoot%\System32\cmd.exe /c start "" '
r"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe "
r"-NoProfile -NonInteractive -ExecutionPolicy Bypass "
rf"-File %SystemDrive%\{FIRSTBOOT_DIR}\firstboot.ps1"
)
CLEANUP_MODES = ("standard", "full")
# who restarts the VM between the phases of the first boot: the guest itself, or
# XO, which follows the first boot and starts the VM again once it has stopped
REBOOT_MODES = ("guest", "xo")
ASSETS = Path(__file__).resolve().parent / "assets" / "windows"
# -- pure helpers (unit tested) --------------------------------------------
def is_hibernated(system_volume: Path) -> bool:
"""hiberfil.sys starting with HIBR/RSTR holds a resumable session (or Fast Startup)."""
hiberfil = find_ci(system_volume, "hiberfil.sys")
if hiberfil is None:
return False
try:
with hiberfil.open("rb") as handle:
signature = handle.read(4).lower()
except OSError:
return False
return signature in (b"hibr", b"rstr")
def windows_version(software: Hive) -> dict:
node = software.node("Microsoft", "Windows NT", "CurrentVersion")
if node is None:
return {}
build_text = software.string(node, "CurrentBuildNumber") or software.string(node, "CurrentBuild") or "0"
build = int(build_text) if build_text.isdigit() else 0
product = software.string(node, "ProductName") or "Windows"
installation = software.string(node, "InstallationType") or ""
# Windows 11 still reports "Windows 10" as ProductName
if build >= 22000 and installation == "Client":
product = product.replace("Windows 10", "Windows 11")
return {
"name": product,
"build": build,
"edition": software.string(node, "EditionID"),
"installation_type": installation,
}
def current_control_set(system: Hive) -> int:
select = system.node("Select")
number = system.dword(select, "Current") if select is not None else None
node = system.node(f"ControlSet{number or 1:03d}")
if node is None:
raise DockingError("no_control_set", "cannot find the current control set in the SYSTEM hive")
return node
def control_sets(system: Hive) -> list[int]:
"""Every ControlSetNNN (current, Last Known Good, failed...)."""
return [node for name, node in system.children(system.root()) if re.fullmatch(r"ControlSet\d{3}", name, re.I)]
def services_of(system: Hive, control_set: int) -> int | None:
return system.child(control_set, "Services")
def is_vmware_service(system: Hive, name: str, node: int) -> bool:
"""VMware Tools component? Conservative on purpose: security agents and
other products from the VMware/Broadcom catalog must not be caught."""
if name.lower() in {k.lower() for k in KNOWN_VMWARE_SERVICES}:
return True
image = (system.string(node, "ImagePath") or "").lower()
if TOOLS_PATH_MARKER in image:
return True
if system.dword(node, "Type") not in (SERVICE_KERNEL_DRIVER, SERVICE_FS_DRIVER):
return False
label = " ".join(filter(None, (system.string(node, "DisplayName"), system.string(node, "Description"))))
return "vmware" in label.lower() and (name.lower().startswith("vm") or "\\vm" in image)
def vmware_services(system: Hive, services: int) -> list[str]:
"""VMware components still enabled (Start != 4)."""
found = []
for name, node in system.children(services):
if name == FIRSTBOOT_SERVICE or system.dword(node, "Start") == 4:
continue
if is_vmware_service(system, name, node):
found.append(name)
return sorted(found, key=str.lower)
def enable_boot_drivers(system: Hive, services: int, names=IDE_SERVICES) -> list[str]:
changed = []
for name in names:
node = system.child(services, name)
if node is None:
continue
system.set_dword(node, "Start", 0)
# Windows 8+ can keep a per-boot-config override disabling the driver
override = system.child(node, "StartOverride")
if override is not None:
system.delete_child(override)
changed.append(name)
return changed
def disable_services(system: Hive, services: int, names) -> list[str]:
changed = []
for name in names:
node = system.child(services, name)
if node is not None and system.dword(node, "Start") != 4:
system.set_dword(node, "Start", 4)
changed.append(name)
return changed
def _child_path(system: Hive, node: int | None, *names: str) -> int | None:
for name in names:
if node is None:
return None
node = system.child(node, name)
return node
def rtc_timeoffset(system: Hive, control_set: int) -> int | None:
"""Offset of the RTC from UTC, in seconds, as Windows expects it (XAPI platform:timeoffset).
Windows keeps the RTC in local time, unless RealTimeIsUniversal is set.
XCP-ng starts the RTC at UTC + platform:timeoffset (0 by default), so
without it the clock is off by the time zone until time sync fixes it.
ActiveTimeBias is UTC - local time, in minutes, at the last shutdown.
"""
tz = _child_path(system, control_set, "Control", "TimeZoneInformation")
if tz is None:
return None
if system.dword(tz, "RealTimeIsUniversal") == 1:
return 0
bias = system.dword(tz, "ActiveTimeBias")
if bias is None:
bias = system.dword(tz, "Bias")
if bias is None:
return None
if bias >= 1 << 31: # REG_DWORD holding a negative number
bias -= 1 << 32
return -bias * 60
def _split_dns(value: str | None) -> list[str]:
return [v for v in re.split(r"[,\s]+", value or "") if v]
def network_adapters(system: Hive, control_set: int) -> list[dict]:
"""TCP/IP configuration of the physical (PCI) adapters, i.e. the VMware ones.
After the migration every PCI adapter is new: the configurations found
here all belong to adapters that no longer exist. Software adapters
(loopback, VPN...) are left out.
"""
adapters_class = _child_path(system, control_set, "Control", "Class", NET_CLASS)
interfaces = _child_path(system, control_set, "Services", "Tcpip", "Parameters", "Interfaces")
if adapters_class is None or interfaces is None:
return []
adapters = []
for _, node in system.children(adapters_class):
component = (system.string(node, "ComponentId") or "").lower()
guid = system.string(node, "NetCfgInstanceId")
if not component.startswith("pci\\") or not guid:
continue
config = system.child(interfaces, guid)
if config is None:
continue
dhcp = system.dword(config, "EnableDHCP")
addresses = [a for a in system.strings(config, "IPAddress") if a != "0.0.0.0"]
masks = [m for m in system.strings(config, "SubnetMask") if m != "0.0.0.0"]
adapters.append({
"adapter": system.string(node, "DriverDesc") or component,
"vmware": "ven_15ad" in component,
"guid": guid,
"dhcp": dhcp != 0,
"addresses": addresses,
"masks": masks,
"gateways": system.strings(config, "DefaultGateway"),
"dns": _split_dns(system.string(config, "NameServer")),
})
return adapters
def clear_static_config(system: Hive, control_set: int, guids) -> int:
"""Switch the old adapters to DHCP so their addresses can be reused."""
interfaces = _child_path(system, control_set, "Services", "Tcpip", "Parameters", "Interfaces")
if interfaces is None:
return 0
cleared = 0
for guid in guids:
config = system.child(interfaces, guid)
if config is None:
continue
system.set_dword(config, "EnableDHCP", 1)
system.set_strings(config, "IPAddress", ["0.0.0.0"])
system.set_strings(config, "SubnetMask", ["0.0.0.0"])
system.set_strings(config, "DefaultGateway", [])
system.set_string(config, "NameServer", "")
cleared += 1
return cleared
def firstboot_state(directory: Path | None, log_lines: int = 5) -> dict | None:
"""State left by firstboot.ps1 in \\xo-docking\\, read offline.
The first boot can only report through xenstore once the PV drivers are
loaded: this is how XO learns why it did not finish.
"""
if directory is None or not (directory / "firstboot.ps1").is_file():
return None
def read(name: str) -> str:
path = find_ci(directory, name)
try:
return path.read_text(errors="replace").strip() if path else ""
except OSError:
return ""
config = dict(re.findall(r"^\s*([^#=\s]+)\s*=\s*(.*?)\s*$", read("config.ini"), re.M))
max_attempts = int(config.get("max_attempts", "3")) if config.get("max_attempts", "3").isdigit() else 3
phase = read("phase") or "drivers"
attempts = int(read("attempts")) if read("attempts").isdigit() else 0
if phase == "done" or find_ci(directory, "firstboot.done"):
status = "done"
elif attempts == 0:
status = f"{phase}:pending" # never started, or a new phase not started yet
elif attempts >= max_attempts:
status = f"{phase}:failed"
else:
status = f"{phase}:error"
log_tail = [line[:200] for line in read("firstboot.log").splitlines()[-log_lines:]]
return {"status": status, "phase": phase, "attempts": attempts, "log": log_tail}
def pv_drivers_installed(system: Hive, services: int) -> bool:
"""The Xen PV storage drivers are installed and enabled."""
nodes = [system.child(services, name) for name in PV_DRIVER_SERVICES]
return all(node is not None and system.dword(node, "Start") != 4 for node in nodes)
def match_packages(files: list[str], patterns: list[str]) -> dict[str, str | None]:
"""First file (relative path, in sorted order) whose name matches each pattern.
Case-insensitive, on the file name only, like Get-ChildItem -Filter in firstboot.ps1.
"""
names = sorted(files, key=str.lower)
return {
pattern: next((f for f in names if fnmatch.fnmatch(f.rsplit("/", 1)[-1].lower(), pattern.lower())), None)
for pattern in patterns
}
def install_firstboot_service(system: Hive, services: int) -> None:
node = system.add_child(services, FIRSTBOOT_SERVICE)
system.set_dword(node, "Type", 0x10) # own process
system.set_dword(node, "Start", 2) # automatic
system.set_dword(node, "ErrorControl", 0) # never block boot
system.set_string(node, "ImagePath", FIRSTBOOT_IMAGE_PATH, expand=True)
system.set_string(node, "DisplayName", "XO docking first boot")
system.set_string(node, "Description", "Installs Xen PV drivers after a migration, then removes itself")
system.set_string(node, "ObjectName", "LocalSystem")
def parse_windows_nics(value: str | None) -> dict[str, str]:
""""00:0c:29:aa:bb:cc={GUID};..." -> {mac: guid}, invalid entries dropped.
XO builds it by matching its vSphere capture (MAC, addresses) with the
`network` of the report (GUID of the old adapter, addresses).
"""
result = {}
for item in (value or "").split(";"):
mac, sep, guid = item.strip().partition("=")
mac = mac.strip().lower().replace("-", ":")
if sep and MAC_RE.match(mac) and GUID_RE.match(guid.strip()):
result[mac] = guid.strip().lower()
return result
def map_static_adapters(static: list[dict], live_macs: list[str],
given: dict[str, str]) -> tuple[list[tuple[str, dict]], list[dict]]:
"""Old adapters with a static address -> MAC of the VIF that replaces them.
Automatic for a single static adapter on a VM with a single NIC; otherwise
from XO's mapping. Returns the (mac, adapter) pairs and the unmapped adapters.
"""
by_guid = {a["guid"].lower(): a for a in static}
if given:
mapped = [(mac, by_guid[guid]) for mac, guid in given.items() if guid in by_guid]
elif len(static) == 1 and len(live_macs) == 1:
mapped = [(live_macs[0], static[0])]
else:
mapped = []
done = {id(adapter) for _, adapter in mapped}
return mapped, [a for a in static if id(a) not in done]
def network_config(mapped: list[tuple[str, dict]]) -> bytes:
"""network.ini for firstboot.ps1: one adapter per line, mac|addresses|masks|gateways|dns."""
lines = ["|".join([mac, ",".join(a["addresses"]), ",".join(a["masks"]),
",".join(a["gateways"]), ",".join(a["dns"])]) for mac, a in mapped]
return ("\r\n".join(lines) + "\r\n").encode("ascii")
def firstboot_msi_patterns(build: int, patterns: list[str] | None) -> list[str]:
"""Packages the first boot installs: none by default below PV9_MIN_BUILD, where the
XCP-ng 9.x MSI fails and would stop the first boot before the static IP is reapplied."""
if patterns is not None:
return list(patterns) # given by XO (windows-msi-patterns), e.g. older tools
return [] if build < PV9_MIN_BUILD else list(DEFAULT_MSI_PATTERNS)
def firstboot_config(msi_patterns: list[str], cleanup: str, reboot: str = "guest", max_attempts: int = 3) -> bytes:
lines = [
f"msi_patterns={';'.join(msi_patterns)}",
f"cleanup={cleanup}",
f"reboot={reboot}",
f"max_attempts={max_attempts}",
]
return ("\r\n".join(lines) + "\r\n").encode("ascii")
# -- guest -------------------------------------------------------------------
class WindowsGuest:
kind = "windows"
def __init__(self, candidate: OsCandidate, mounts: Mounts, report: Report, readonly: bool,
msi_patterns: list[str] | None = None, cleanup: str = "standard", tools: Path | None = None,
windows_nics: dict[str, str] | None = None, live_macs: list[str] | None = None,
reboot: str = "guest"):
self.windows_nics = windows_nics or {}
# VIF MACs as the live system sees them: they are the MACs of the VMware NICs
self.live_macs = live_macs if live_macs is not None else [mac for mac, _ in live_nics().values()]
self.static_config: list[tuple[str, dict]] = [] # reapplied by the first boot
self.candidate = candidate
self.mounts = mounts
self.report = report
self.readonly = readonly
self.given_msi_patterns = msi_patterns # None: default, depends on the build
self.msi_patterns = list(msi_patterns if msi_patterns is not None else DEFAULT_MSI_PATTERNS)
self.cleanup = cleanup
self.reboot = reboot
self.tools = tools # mounted guest tools disk
self.packages: list[Path] = [] # copied into \xo-docking\packages\
self.root: Path | None = None
self.system: Hive | None = None
self.system_path: Path | None = None
self.os_ids = ["windows"]
def inspect(self) -> None:
if self.cleanup not in CLEANUP_MODES:
self.report.error("invalid_cleanup_mode",
f"unknown cleanup mode {self.cleanup!r}, expected one of {', '.join(CLEANUP_MODES)}")
return
if self.reboot not in REBOOT_MODES:
self.report.error("invalid_reboot_mode",
f"unknown reboot mode {self.reboot!r}, expected one of {', '.join(REBOOT_MODES)}")
return
dev = self.candidate.device
self.root = self.mounts.mount(dev.path, "ntfs", readonly=self.readonly)
if is_hibernated(self.root):
self.report.error("windows_hibernated",
"Windows is hibernated or was stopped with Fast Startup; "
"do a full shutdown on the source (shutdown /s /f /t 0)")
return
config = find_ci(self.root, "Windows", "System32", "config")
software_path = find_ci(config, "SOFTWARE")
self.system_path = find_ci(config, "SYSTEM")
software = Hive(software_path)
try:
version = windows_version(software)
finally:
software.close()
self.report.os = {"kind": "windows", "root": dev.path, "cleanup": self.cleanup, **version}
if version.get("build", 0) < MIN_BUILD:
self.report.error("unsupported_windows_version",
f"Windows build {version.get('build')} is not supported "
f"(minimum {MIN_BUILD}, Windows 7 SP1 / Server 2008 R2 SP1)")
return
self.msi_patterns = firstboot_msi_patterns(version.get("build", 0), self.given_msi_patterns)
if version.get("build", 0) < PV9_MIN_BUILD:
self.report.warning("pv_drivers_unsupported",
f"the XCP-ng 9.x PV drivers need build {PV9_MIN_BUILD} or later "
"(Windows 10 1607 / Server 2016): the VM runs on emulated disk and NIC, "
"the first boot installs no PV driver unless windows-msi-patterns "
"points to older tools", build=version.get("build"), min_build=PV9_MIN_BUILD)
self._check_firstboot()
self.system = Hive(self.system_path, write=not self.readonly)
current = current_control_set(self.system)
self.control_sets = control_sets(self.system)
self.report.os["control_sets"] = len(self.control_sets)
self.report.os["rtc_timeoffset"] = rtc_timeoffset(self.system, current)
self._plan_ide(current)
self._plan_vmware(current)
self._plan_network(current)
self._plan_firstboot(current)
self.report.plan("commit_registry", "back up then write the SYSTEM hive", self._commit)
def finalize(self) -> None:
pass
def close(self) -> None:
"""Release the hive before the volume is unmounted."""
if self.system is not None:
self.system.close()
def write_report(self, report: Report) -> None:
target = self.root / FIRSTBOOT_DIR
target.mkdir(exist_ok=True)
(target / "report.json").write_text(report.full_json())
log_file = Path("/run/xo-docking/xo-docking.log")
if log_file.exists():
shutil.copyfile(log_file, target / "xo-docking.log")
# -- planning --
def _for_each_services(self, function) -> None:
for control_set in self.control_sets:
services = services_of(self.system, control_set)
if services is not None:
function(services)
def _plan_ide(self, current: int) -> None:
services = services_of(self.system, current)
present = [n for n in IDE_SERVICES if self.system.child(services, n) is not None]
if not {"intelide", "pciide"} & set(present):
self.report.error("no_ide_driver", "neither intelide nor pciide exists in this Windows")
return
self.report.plan("enable_ide_boot",
f"start {', '.join(present)} at boot (emulated IDE disk), "
f"in {len(self.control_sets)} control set(s)",
lambda: self._for_each_services(lambda s: enable_boot_drivers(self.system, s, present)))
def _plan_vmware(self, current: int) -> None:
names = vmware_services(self.system, services_of(self.system, current))
self.report.os["vmware_services"] = names
if names:
self.report.plan("disable_vmware_services",
f"disable {len(names)} VMware service(s) and driver(s) in "
f"{len(self.control_sets)} control set(s): {', '.join(names)}",
lambda: self._for_each_services(lambda s: disable_services(self.system, s, names)),
critical=False)
def _plan_network(self, current: int) -> None:
adapters = network_adapters(self.system, current)
self.report.network = adapters
static = [a for a in adapters if not a["dhcp"] and a["addresses"]]
if not static:
return
addresses = [ip for a in static for ip in a["addresses"]]
self.report.warning("windows_static_ip",
"static IP configuration of the old adapter(s) captured in the report: "
+ ", ".join(addresses), addresses=addresses)
self.static_config, unmapped = map_static_adapters(static, self.live_macs, self.windows_nics)
if unmapped:
# XO reapplies them with VIF.configure_ipv4 once the agent is up (no DNS)
self.report.warning("windows_static_ip_unmapped",
"no MAC for the old adapter(s) "
+ ", ".join(ip for a in unmapped for ip in a["addresses"])
+ ": pass windows-nics to have the first boot reapply them",
guids=[a["guid"] for a in unmapped])
guids = [a["guid"] for a in static]
def apply() -> None:
for control_set in self.control_sets:
clear_static_config(self.system, control_set, guids)
self.report.plan("clear_old_static_ip",
f"switch {len(static)} old adapter(s) to DHCP so their addresses can be reused",
apply, critical=False)
def _check_firstboot(self) -> None:
"""Report the state of a previous first boot (read before docking resets it)."""
state = firstboot_state(find_ci(self.root, FIRSTBOOT_DIR))
if state is None:
return
self.report.os["firstboot"] = state["status"]
if state["status"] == "done":
self.report.info("windows_firstboot_done", "the previous first boot finished", log=state["log"])
else:
self.report.warning("windows_firstboot_incomplete",
f"the previous first boot did not finish: {state['status']} "
f"after {state['attempts']} attempt(s)",
attempts=state["attempts"], log=state["log"])
def _plan_firstboot(self, current: int) -> None:
# dock run again on a migrated VM (XO retry): do not reinstall the drivers
if (self.report.os.get("firstboot") == "done"
and pv_drivers_installed(self.system, services_of(self.system, current))):
self.report.info("windows_firstboot_already_done",
"a previous first boot installed the Xen PV drivers: it is not scheduled again")
return
patterns = ";".join(self.msi_patterns)
if not self.msi_patterns:
source = "none"
elif self.tools is None:
self.report.warning("windows_tools_disk_missing",
f"no guest tools disk: the first boot will have no package matching {patterns} "
"to install")
source = "missing, no guest tools disk"
else:
files = [str(p.relative_to(self.tools)) for p in self.tools.rglob("*") if p.is_file()]
matches = match_packages(files, self.msi_patterns)
missing = [pattern for pattern, found in matches.items() if found is None]
if missing:
self.report.error("windows_msi_not_found",
"no file matching " + ", ".join(missing) + " on the guest tools disk",
patterns=missing)
found = list(dict.fromkeys(f for f in matches.values() if f))
self.packages = [self.tools / f for f in found]
source = "the guest tools disk (" + ", ".join(p.name for p in self.packages) + ")"
network = ""
if self.static_config:
network = ", static IP configuration of " + ", ".join(mac for mac, _ in self.static_config)
self.report.plan("install_firstboot",
f"install the first-boot script ({self.cleanup} cleanup, {self.reboot} reboot, Xen PV drivers: {source}"
f"{network})", self._apply_firstboot)
# -- apply --
def _apply_firstboot(self) -> None:
target = self.root / FIRSTBOOT_DIR
target.mkdir(exist_ok=True)
shutil.copyfile(ASSETS / "firstboot.ps1", target / "firstboot.ps1")
(target / "config.ini").write_bytes(firstboot_config(self.msi_patterns, self.cleanup, self.reboot))
network = target / "network.ini"
if self.static_config:
network.write_bytes(network_config(self.static_config))
else:
network.unlink(missing_ok=True) # previous run of the ISO
packages = target / PACKAGES_DIR
shutil.rmtree(packages, ignore_errors=True) # previous run of the ISO
if self.packages:
packages.mkdir()
for package in self.packages:
shutil.copyfile(package, packages / package.name)
for stale in ("phase", "attempts", "firstboot.done"):
(target / stale).unlink(missing_ok=True) # previous run of the ISO
self._for_each_services(lambda s: install_firstboot_service(self.system, s))
def _commit(self) -> None:
backup = self.system_path.with_name(self.system_path.name + ".xo-docking.bak")
# the hive before the first docking: a later run would back up a hive it already modified
if not backup.exists():
shutil.copyfile(self.system_path, backup)
self.system.commit()
self.system.close()

View File

@@ -1,71 +0,0 @@
"""xenstore access from inside the guest.
XO -> ISO: VM.xenstore_data entries, readable at vm-data/xo-docking/*
ISO -> XO: written under vm-data/xo-docking/out/*. XAPI only syncs the guest's
vm-data/ subtree back into VM.xenstore_data (not data/), and keeps it after
the VM halts.
Paths are relative to the domain's home (/local/domain/<domid>/).
"""
from __future__ import annotations
import logging
import shutil
import subprocess
from pathlib import Path
log = logging.getLogger("xo-docking")
CONFIG_PREFIX = "vm-data/xo-docking"
STATUS_PREFIX = "vm-data/xo-docking/out"
class Xenstore:
def __init__(self, enabled: bool):
self.enabled = enabled
@classmethod
def detect(cls) -> "Xenstore":
has_tools = shutil.which("xenstore-read") is not None
has_device = Path("/dev/xen/xenbus").exists() or Path("/proc/xen/xenbus").exists()
if not (has_tools and has_device):
log.warning("xenstore unavailable: status will only be logged")
return cls(has_tools and has_device)
def read(self, path: str) -> str | None:
if not self.enabled:
return None
# -R: raw value, otherwise \ and non-printable bytes come back escaped
proc = subprocess.run(["xenstore-read", "-R", path], capture_output=True, text=True)
if proc.returncode != 0:
return None
return proc.stdout.strip()
def write(self, path: str, value: str) -> bool:
if not self.enabled:
log.info("xenstore (disabled) %s = %s", path, value)
return False
# -R: without it xenstore-write unescapes the value (\\ -> \, \xNN -> byte),
# and a JSON report with a Windows path (\EFI\BOOT) becomes invalid
proc = subprocess.run(["xenstore-write", "-R", path, value], capture_output=True, text=True)
if proc.returncode != 0:
log.error("xenstore-write %s failed: %s", path, proc.stderr.strip())
return False
return True
def remove(self, path: str) -> bool:
if not self.enabled:
return False
proc = subprocess.run(["xenstore-rm", path], capture_output=True, text=True)
return proc.returncode == 0
def config(self, key: str) -> str | None:
return self.read(f"{CONFIG_PREFIX}/{key}")
def status(self, key: str, value: str) -> bool:
return self.write(f"{STATUS_PREFIX}/{key}", value)
def clear_status(self) -> None:
"""Remove the outputs of a previous run (they persist in VM.xenstore_data)."""
self.remove(STATUS_PREFIX)

View File

@@ -1,3 +0,0 @@
#!/bin/sh
export PYTHONPATH="/usr/lib/xo-docking${PYTHONPATH:+:$PYTHONPATH}"
exec python3 -m xo_docking "$@"

View File

@@ -1,4 +0,0 @@
live-boot
live-config
live-config-systemd
systemd-sysv

View File

@@ -1,22 +0,0 @@
# runtime of the docking tool
python3
python3-hivex
xenstore-utils
# keep-running: live-config autologin on the console needs the live user
sudo
user-setup
# filesystems and storage stacks found on migrated VMs
btrfs-progs
dosfstools
e2fsprogs
lvm2
mdadm
ntfs-3g
xfsprogs
# lsinitramfs: read the guest's initramfs when it cannot be rebuilt (read-only root)
lz4
xz-utils
zstd

View File

@@ -1,21 +0,0 @@
{
"private": true,
"name": "@xen-orchestra/autopilot-docking",
"version": "0.0.0",
"description": "Live ISO that prepares a VM imported from another hypervisor to boot on XCP-ng",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/autopilot-docking",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/autopilot-docking",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"engines": {
"node": ">=18"
}
}

View File

@@ -1,33 +0,0 @@
#!/bin/sh
# Build, check and version the docking ISO. Run as root, from any folder: it works in the package folder.
# Usage: scripts/build.sh
# XO_DOCKING_VERSION=1.2.0 scripts/build.sh # release build (CI sets it from the tag)
# Produces dist/vates-autopilot-docking-<YYYYMMDDTHHMMSSZ>-<commit or release version>.iso and its .sha256.
# The build date and time first, in UTC, for every build: XO uses the last ISO of the pool by name.
set -eu
cd "$(dirname "$0")/.."
# root does not own the checkout: let git read it anyway
# '*' and not "$PWD": the checkout is the root of the monorepo, not this package folder
git() { command git -c safe.directory='*' "$@"; }
version="$(date -u +%Y%m%dT%H%M%SZ)-${XO_DOCKING_VERSION:-}"
if [ -z "${XO_DOCKING_VERSION:-}" ]; then
version="$version$(git rev-parse --short=8 HEAD)"
# only this package: a change elsewhere in the monorepo does not end up in the ISO
[ -z "$(git status --porcelain --untracked-files=no -- .)" ] || version="$version-dirty"
fi
echo "$version" > config/includes.chroot/usr/lib/xo-docking/VERSION
lb config
lb build
scripts/check-iso.sh live-image-amd64.hybrid.iso
name="vates-autopilot-docking-$version.iso"
mkdir -p dist
mv live-image-amd64.hybrid.iso "dist/$name"
(cd dist && sha256sum "$name" > "$name.sha256")
[ -z "${GITHUB_OUTPUT:-}" ] || echo "version=$version" >> "$GITHUB_OUTPUT"
echo "build: dist/$name"

View File

@@ -1,68 +0,0 @@
#!/bin/sh
# Sanity checks on a built ISO: boot timeouts, boot menu background and presence of the docking tool.
# Usage: scripts/check-iso.sh path/to/image.iso
set -eu
iso="$1"
fail() { echo "check-iso: $*" >&2; exit 1; }
[ -s "$iso" ] || fail "$iso is missing or empty (did lb build fail?)"
# next to the ISO rather than in /tmp: the squashfs is several hundred MB and
# /tmp is often a small tmpfs in CI containers
work="$(mktemp -d "$(dirname "$iso")/check-iso.XXXXXX")"
# extracted files are read-only
trap 'chmod -R u+w "$work" 2>/dev/null; rm -rf "$work"' EXIT
if ! xorriso -osirrox on -indev "$iso" \
-extract /boot/grub "$work/grub" \
-extract /isolinux "$work/isolinux" \
-extract /live/filesystem.squashfs "$work/fs.squashfs" > "$work/xorriso.log" 2>&1; then
grep -v 'UPDATE' "$work/xorriso.log" | tail -n 20 >&2
df -h "$work" >&2
fail "cannot extract boot files from $iso"
fi
# grub (UEFI): at least one timeout, none longer than 1 s
grep -rhoE 'set[[:space:]]+timeout=[0-9]+' "$work/grub" > "$work/grub.timeouts" \
|| fail "no timeout found in the grub configuration"
if grep -vqE 'timeout=(0|1)$' "$work/grub.timeouts"; then
fail "grub timeout too long: $(sort -u "$work/grub.timeouts" | tr '\n' ' ')"
fi
# syslinux (BIOS): "timeout 0" waits forever
grep -rhioE '^[[:space:]]*timeout[[:space:]]+[0-9]+' "$work/isolinux" > "$work/isolinux.timeouts" \
|| fail "no timeout found in the isolinux configuration"
if grep -qiE 'timeout[[:space:]]+0$' "$work/isolinux.timeouts"; then
fail "isolinux waits forever (timeout 0)"
fi
# boot menu background: ours, at the size each boot loader expects (PNG width and
# height are the big-endian 32-bit integers at offset 16 of the IHDR chunk)
png_size() { od -An -tu1 -j16 -N8 "$1" | awk '{ print $1*16777216+$2*65536+$3*256+$4 "x" $5*16777216+$6*65536+$7*256+$8 }'; }
for splash in isolinux/splash.png:640x480 grub/splash.png:800x600; do
file="$work/${splash%%:*}"
[ -s "$file" ] || fail "missing boot menu background: ${splash%%:*}"
size="$(png_size "$file")"
[ "$size" = "${splash#*:}" ] || fail "${splash%%:*} is $size, ${splash#*:} expected"
done
grep -q '^title-text: ""$' "$work/grub/live-theme/theme.txt" \
|| fail "the grub menu title is drawn over the boot menu background"
! grep -q '^menu title Boot menu' "$work/isolinux/menu.cfg" \
|| fail "the isolinux menu title is drawn over the boot menu background"
# the docking tool and its runtime are in the live system
listing="$work/fs.list"
unsquashfs -l "$work/fs.squashfs" > "$listing"
for path in \
usr/local/bin/xo-docking \
usr/lib/xo-docking/VERSION \
usr/lib/xo-docking/xo_docking/__main__.py \
usr/lib/xo-docking/xo_docking/assets/windows/firstboot.ps1 \
etc/systemd/system/multi-user.target.wants/xo-docking.service; do
grep -q "/$path\$" "$listing" || fail "missing in the live system: /$path"
done
for tool in python3 xenstore-read xenstore-write xenstore-rm ntfs-3g; do
grep -qE "/(usr/)?s?bin/$tool\$" "$listing" || fail "missing tool in the live system: $tool"
done
echo "check-iso: $iso OK"

View File

@@ -1,54 +0,0 @@
"""Test helpers: import path, fake guest trees, empty registry hive."""
import base64
import json
import logging
import sys
import zlib
from pathlib import Path
PACKAGE_DIR = Path(__file__).resolve().parents[1] / "config/includes.chroot/usr/lib/xo-docking"
sys.path.insert(0, str(PACKAGE_DIR))
# keep test output readable
logging.getLogger("xo-docking").addHandler(logging.NullHandler())
# Empty registry hive ("minimal" image from the hivex test suite), zlib + base64.
MINIMAL_HIVE = (
"eNrt2LtKA0EUBuB/Ehdh8bJIiggWK1mwEWu74JUIwUiIgpBK8RIWg0QfYEsfwMZCsPA9tBQrH8EH8AlSOM7ZmSWuScBS8P+W"
"2Tln+PdCSLPTOzk7hQJkvN+83U8/vippPVhSh1IEtm6jao429rGLDTNfo4cOjhGn9QUuzbyJLWxjzWTqaIH+qqfD1T5/BSIi"
"IiIiov/h/KjThfu+z2T7AFI/aK278TLCpfW7bO15cpDVTlYnbh71rCkzoijaazZajZXazoGsvXxqfRUDcl3i9hsWlMwTt9IE"
"yl5bMqOAD/hpV8a86VQhbTzZn1iU2i9Lnwytj8hXxuQrLl/FbC4fDvLB0Lp5uzRv39Wbk7Nfstnva2Pvm8sW5fQzk78PUJvh"
"f5eIiIiIiIh+7wuThkCS"
)
def write_empty_hive(path: Path) -> Path:
path.write_bytes(zlib.decompress(base64.b64decode(MINIMAL_HIVE)))
return path
def make_tree(root: Path, files: dict) -> Path:
"""Create files ({"etc/fstab": "..."}) and symlinks ({"lib": ("symlink", "usr/lib")})."""
for relpath, content in files.items():
path = root / relpath
path.parent.mkdir(parents=True, exist_ok=True)
if isinstance(content, tuple) and content[0] == "symlink":
path.symlink_to(content[1])
elif isinstance(content, bytes):
path.write_bytes(content)
else:
path.write_text(content)
return root
def lsblk_json(*nodes) -> str:
return json.dumps({"blockdevices": list(nodes)})
def dev(name, type_="part", children=None, **fields):
node = {"name": name, "path": f"/dev/{name}", "type": type_, "rm": False, "size": 1 << 30}
node.update(fields)
if children:
node["children"] = children
return node

View File

@@ -1,88 +0,0 @@
import unittest
from unittest import mock
from helpers import dev, lsblk_json
from xo_docking import __main__ as main
from xo_docking.blockdev import activate_storage, guest_disks, parse_lsblk
from xo_docking.report import Report
from xo_docking.util import DockingError
class ParseLsblkTest(unittest.TestCase):
def setUp(self):
text = lsblk_json(
dev("sr0", "rom", rm=True, fstype="iso9660"),
dev("loop0", "loop", fstype="squashfs"),
dev("xvdb", "disk", pttype="gpt", children=[
dev("xvdb1", fstype="LVM2_member", children=[
dev("vg-data", "lvm", path="/dev/mapper/vg-data", fstype="xfs", uuid="U-DATA"),
]),
]),
dev("xvda", "disk", pttype="gpt", children=[
dev("xvda1", fstype="vfat", parttype="C12A7328-F81F-11D2-BA4B-00A0C93EC93B"),
dev("xvda2", fstype="ext4", uuid="U-ROOT"),
]),
dev("nvme0n1", "disk", children=[dev("nvme0n1p3", fstype="ext4")]),
)
self.devices = {d.name: d for d in parse_lsblk(text)}
def test_hierarchy(self):
self.assertEqual(self.devices["vg-data"].disk, "xvdb")
self.assertEqual(self.devices["vg-data"].parent, "xvdb1")
self.assertEqual(self.devices["vg-data"].path, "/dev/mapper/vg-data")
self.assertEqual(self.devices["xvda2"].disk, "xvda")
def test_partition_number(self):
self.assertEqual(self.devices["xvda2"].partition_number(), 2)
self.assertEqual(self.devices["nvme0n1p3"].partition_number(), 3)
self.assertIsNone(self.devices["vg-data"].partition_number())
def test_esp(self):
self.assertTrue(self.devices["xvda1"].is_esp)
self.assertFalse(self.devices["xvda2"].is_esp)
def test_guest_disks_order_and_filtering(self):
names = [d.name for d in guest_disks(list(self.devices.values()))]
self.assertEqual(names, ["xvda", "xvdb", "nvme0n1"])
def test_guest_tools_disk_is_not_a_guest_disk(self):
text = lsblk_json(dev("xvdz", "disk", fstype="iso9660", label="XCP-ng Tools"),
dev("xvda", "disk", children=[dev("xvda1", fstype="ext4")]))
devices = parse_lsblk(text)
self.assertEqual([d.name for d in guest_disks(devices)], ["xvda"])
self.assertEqual([d.name for d in devices if d.is_medium], ["xvdz"])
def test_disk_order_beyond_z(self):
text = lsblk_json(dev("xvdaa", "disk"), dev("xvdz", "disk"), dev("xvda", "disk"))
self.assertEqual([d.name for d in guest_disks(parse_lsblk(text))], ["xvda", "xvdz", "xvdaa"])
class ActivateStorageTest(unittest.TestCase):
def test_returns_the_failures(self):
def run(cmd, **kwargs):
if cmd[0] == "vgchange":
raise DockingError("command_failed", "vgchange exited with status 5: metadata unreadable")
with mock.patch("xo_docking.blockdev.run", side_effect=run):
self.assertEqual(activate_storage(readonly=True),
["vgchange exited with status 5: metadata unreadable"])
def test_reported_when_no_os_is_found(self):
report = Report("diagnose", "dev")
disks = parse_lsblk(lsblk_json(dev("xvda", "disk")))
with mock.patch.object(main.efi, "firmware", return_value="bios"), \
mock.patch.object(main, "activate_storage", return_value=["vgchange exited with status 5"]), \
mock.patch.object(main, "list_block_devices", return_value=[]), \
mock.patch.object(main, "guest_disks", return_value=disks), \
mock.patch.object(main, "find_tools", return_value=None), \
mock.patch.object(main, "probe", return_value=[]):
main.execute(report, mock.Mock(), None, "standard", {}, {})
self.assertEqual([(f.level, f.code, f.message) for f in report.findings], [
("warning", "storage_activation_failed", "vgchange exited with status 5"),
("error", "no_os_found", "no Linux or Windows installation found on the disks"),
])
if __name__ == "__main__":
unittest.main()

View File

@@ -1,70 +0,0 @@
import re
import unittest
from helpers import PACKAGE_DIR
SCRIPT = PACKAGE_DIR / "xo_docking/assets/windows/firstboot.ps1"
# "$name:" in a double-quoted string is parsed as a scope-qualified variable
# ($env:X): anything but a real scope is a parse error that rejects the whole script.
SCOPES = {"env", "script", "global", "local", "private", "using", "function", "variable"}
STRING_RE = re.compile(r'"(?:[^"`]|`.)*"')
SCOPED_RE = re.compile(r"\$(\w+):")
def function_body(name: str) -> str:
match = re.search(rf"^function {re.escape(name)}\b.*?^}}$", SCRIPT.read_text(), re.MULTILINE | re.DOTALL)
assert match, f"no function {name} in {SCRIPT.name}"
return match.group(0)
class FirstbootScriptTest(unittest.TestCase):
def test_no_invalid_scoped_variable(self):
errors = []
for lineno, line in enumerate(SCRIPT.read_text().splitlines(), start=1):
if line.lstrip().startswith("#"):
continue
for string in STRING_RE.findall(line):
for name in SCOPED_RE.findall(string):
if name.lower() not in SCOPES:
errors.append(f"line {lineno}: ${name}: (use ${{{name}}}:)")
self.assertEqual(errors, [])
def test_xenstore_classes(self):
text = SCRIPT.read_text()
self.assertIn("'XenProjectXenStoreBase'", text)
self.assertIn("'vm-data/xo-docking/out/firstboot'", text)
def test_reboot_left_to_xo(self):
# a VM started by hand is never left stopped: the guest reboots by default
self.assertIn("'reboot' = 'guest'", function_body("Read-Config"))
body = function_body("Restart-Now")
xo_branch, guest_branch = body.split("if ($script:Config['reboot'] -eq 'xo')")[1].split("exit 0", 1)
# XO starts the VM again: the guest shuts down; otherwise it reboots itself
self.assertIn("shutdown.exe /s ", xo_branch)
self.assertNotIn("shutdown.exe /r", xo_branch)
self.assertIn("shutdown.exe /r ", guest_branch)
self.assertNotIn("shutdown.exe /s", guest_branch)
def test_folder_restricted_before_anything_is_read(self):
body = function_body("Protect-Base")
# by SID: the names of the groups are localized
self.assertIn("'*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F'", body)
self.assertIn("/inheritance:r", body)
main = SCRIPT.read_text().split("# -- main --", 1)[1]
self.assertLess(main.index("Protect-Base"), main.index("Read-Config"))
def test_packages_only_from_the_restricted_folder(self):
body = function_body("Find-Package")
self.assertIn("Join-Path $Base 'packages'", body)
self.assertNotIn("Win32_LogicalDisk", body)
def test_failed_attempt_retried_at_once_when_xo_follows(self):
body = function_body("Stop-Attempt")
retry = body[body.index("elseif ($script:Config['reboot'] -eq 'xo')"):]
# a reboot: a shut down would be taken by XO for the end of the phase
self.assertIn("shutdown.exe /r ", retry)
self.assertNotIn("shutdown.exe /s", body)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,613 +0,0 @@
import io
import os
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from helpers import dev, lsblk_json, make_tree
from xo_docking.blockdev import guest_disks, parse_lsblk
from xo_docking.detect import OsCandidate
from xo_docking.linux import (
VMWARE_UNITS, LinuxGuest, agent_enabled, available_modules, detect_initramfs_tool, disable_units,
enable_unit, extract_agent_tgz,
guest_arch, initramfs_image, missing_initramfs_modules, interface_names, is_unstable_spec, kernel_versions, link_file,
live_nics, map_by_name_spec, map_sd_spec, mountable_entries, package_manager, parse_fstab, parse_nic_map,
parse_os_release, pin_interface_names, rewrite_fstab, rhel_kdump_images, select_agent_package,
selinux_enabled, services_not_started, stale_kdump_initrds, unstable_resume,
)
from xo_docking.report import Report
from xo_docking.util import DockingError, guest_read, resolve_in_root
FSTAB = """\
# /etc/fstab: static file system information.
/dev/sda2 / ext4 errors=remount-ro 0 1
UUID=1234-ABCD /boot/efi vfat umask=0077 0 1
/dev/sdb1\t/data\txfs\tdefaults\t0 2
/dev/sda3 none swap sw 0 0
/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:2:0-part1 /srv ext4 defaults 0 2
proc /proc proc
"""
class FstabTest(unittest.TestCase):
def setUp(self):
devices = parse_lsblk(lsblk_json(
dev("xvda", "disk", children=[
dev("xvda1", uuid="1234-ABCD"), dev("xvda2", uuid="root-uuid"), dev("xvda3", uuid="swap-uuid"),
]),
dev("xvdb", "disk", children=[dev("xvdb1", uuid="data-uuid")]),
))
self.devices = devices
self.disks = guest_disks(devices)
def test_parse(self):
entries = parse_fstab(FSTAB)
self.assertEqual([e.mountpoint for e in entries], ["/", "/boot/efi", "/data", "none", "/srv", "/proc"])
self.assertEqual(entries[0].lineno, 2)
self.assertEqual(entries[-1].options, "defaults")
def test_unstable(self):
self.assertTrue(is_unstable_spec("/dev/sda2"))
self.assertTrue(is_unstable_spec("/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:2:0-part1"))
self.assertFalse(is_unstable_spec("UUID=1234"))
self.assertFalse(is_unstable_spec("/dev/mapper/vg-root"))
self.assertFalse(is_unstable_spec("/dev/xvda1"))
# Ubuntu installs the root LV by its LVM UUID
self.assertFalse(is_unstable_spec("/dev/disk/by-id/dm-uuid-LVM-xCX2aOHDiNpEJsyUKMQcp0BsuKOcAWcN"))
self.assertFalse(is_unstable_spec("/dev/disk/by-id/md-uuid-1234:5678"))
self.assertTrue(is_unstable_spec("/dev/disk/by-id/scsi-36000c29abc-part1"))
def test_map_sd(self):
self.assertEqual(map_sd_spec("/dev/sda2", self.disks, self.devices).name, "xvda2")
self.assertEqual(map_sd_spec("/dev/sdb1", self.disks, self.devices).name, "xvdb1")
self.assertEqual(map_sd_spec("/dev/sdb", self.disks, self.devices).name, "xvdb")
self.assertIsNone(map_sd_spec("/dev/sdc1", self.disks, self.devices))
self.assertIsNone(map_sd_spec("/dev/sda9", self.disks, self.devices))
def test_rewrite_keeps_layout(self):
result = rewrite_fstab(FSTAB, {2: "UUID=root-uuid", 4: "UUID=data-uuid"})
lines = result.splitlines()
self.assertEqual(lines[0], "# /etc/fstab: static file system information.")
self.assertEqual(lines[1], "UUID=root-uuid / ext4 errors=remount-ro 0 1")
self.assertEqual(lines[3], "UUID=data-uuid\t/data\txfs\tdefaults\t0 2")
self.assertTrue(result.endswith("\n"))
class ByNameSpecTest(unittest.TestCase):
"""SLES/YaST "mount by device ID": the VMware names are gone on XCP-ng."""
ROOT = "/dev/disk/by-id/scsi-36000c29abc-part2"
def setUp(self):
self.devices = parse_lsblk(lsblk_json(
dev("xvda", "disk", children=[
dev("xvda1", uuid="boot-uuid"), dev("xvda2", uuid="root-uuid"), dev("xvda3", uuid="swap-uuid"),
]),
dev("xvdb", "disk", children=[dev("xvdb1", uuid="data-uuid")]),
))
self.root = next(d for d in self.devices if d.name == "xvda2")
def map(self, spec, anchor_spec=ROOT, anchor=None):
return map_by_name_spec(spec, anchor_spec, anchor or self.root, self.devices)
def test_same_disk_as_root(self):
self.assertTrue(is_unstable_spec(self.ROOT))
self.assertEqual(self.map("/dev/disk/by-id/scsi-36000c29abc-part3").name, "xvda3")
self.assertEqual(self.map("/dev/disk/by-id/scsi-36000c29abc-part1").name, "xvda1")
self.assertEqual(self.map("/dev/disk/by-id/scsi-36000c29abc").name, "xvda")
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29abc-part9"))
def test_by_path(self):
root = "/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:0:0-part2"
self.assertEqual(self.map("/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:0:0-part3", root).name, "xvda3")
# a by-id alias of the same disk is a different name: not guessed
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29abc-part3", root))
def test_other_disk_unresolved(self):
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29def-part1"))
def test_needs_a_consistent_anchor(self):
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29abc-part3", "UUID=root-uuid"))
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29abc-part3", None))
# the root spec says part2 but the root device is xvda1: do not trust it
xvda1 = next(d for d in self.devices if d.name == "xvda1")
self.assertIsNone(self.map("/dev/disk/by-id/scsi-36000c29abc-part3", anchor=xvda1))
self.assertIsNone(map_by_name_spec("/dev/disk/by-id/scsi-36000c29abc-part3", self.ROOT, None, self.devices))
def test_unstable_resume(self):
self.assertEqual(
unstable_resume('GRUB_CMDLINE_LINUX_DEFAULT="splash=silent resume=/dev/disk/by-id/scsi-3600-part3 quiet"'),
"/dev/disk/by-id/scsi-3600-part3",
)
self.assertEqual(unstable_resume('GRUB_CMDLINE_LINUX_DEFAULT="quiet resume=/dev/sda3"'), "/dev/sda3")
self.assertIsNone(unstable_resume('GRUB_CMDLINE_LINUX_DEFAULT="resume=UUID=9c97 quiet"'))
self.assertIsNone(unstable_resume('GRUB_CMDLINE_LINUX_DEFAULT="splash=silent quiet"'))
class MountableTest(unittest.TestCase):
def test_skips_noauto_and_cdrom(self):
entries = parse_fstab(
"/dev/sda1 / ext4 defaults 0 1\n"
"/dev/sr0 /media/cdrom0 udf,iso9660 user,noauto 0 0\n"
"/dev/sdc1 /mnt/usb ext4 noauto,user 0 0\n"
"/dev/sr1 /media/cdrom1 udf,iso9660 ro 0 0\n"
"/srv/a /srv/b none bind 0 0\n"
"UUID=x /var/lib ext4 defaults 0 2\n"
"UUID=y /var ext4 defaults 0 2\n"
)
self.assertEqual([e.mountpoint for e in mountable_entries(entries)], ["/var", "/var/lib"])
class MountFstabTest(unittest.TestCase):
def test_mount_point_stays_in_the_guest_root(self):
with tempfile.TemporaryDirectory() as tmp:
# an absolute symlink, which the live system would follow out of the guest
root = make_tree(Path(tmp), {"boot": ("symlink", "/guest-boot"), "guest-boot/.keep": ""})
devices = parse_lsblk(lsblk_json(
dev("xvda", "disk", children=[dev("xvda1", fstype="ext4", uuid="U-BOOT")]),
))
mounts = mock.Mock()
guest = LinuxGuest(OsCandidate("linux", devices[1]), devices, guest_disks(devices), mounts,
Report("dock", "dev"), readonly=False)
guest.root = root
guest._mount_fstab(parse_fstab("UUID=U-BOOT /boot ext4 defaults 0 2\n"))
mounts.mount.assert_called_once_with("/dev/xvda1", "ext4", root / "guest-boot", readonly=False,
options=[])
class NetworkTest(unittest.TestCase):
MAC = "00:0c:29:d4:bf:19"
def test_parse_nic_map(self):
self.assertEqual(
parse_nic_map("ens192=00:0C:29:D4:BF:19; ens224=00-0c-29-00-00-01;bad=xyz;;=00:0c:29:00:00:02"),
{"ens192": self.MAC, "ens224": "00:0c:29:00:00:01"},
)
self.assertEqual(parse_nic_map(None), {})
def test_live_nics(self):
with tempfile.TemporaryDirectory() as tmp:
sys_net = make_tree(Path(tmp), {
"drivers/vif/bind": "",
"devices/vif-0/driver": ("symlink", "../../drivers/vif"),
"net/enX0/address": self.MAC.upper() + "\n",
"net/enX0/device": ("symlink", "../../devices/vif-0"),
"net/lo/address": "00:00:00:00:00:00\n",
}) / "net"
self.assertEqual(live_nics(sys_net), {"enX0": (self.MAC, "vif")})
def test_pin_single_nic(self):
nics = {"enX0": (self.MAC, "vif")}
self.assertEqual(pin_interface_names(["ens192"], nics, {}), ({"ens192": (self.MAC, "vif")}, []))
# kernel names are left alone
self.assertEqual(pin_interface_names(["eth0"], nics, {}), ({}, []))
def test_pin_several_nics(self):
nics = {"enX0": (self.MAC, "vif"), "enX1": ("00:0c:29:00:00:01", "vif")}
self.assertEqual(pin_interface_names(["ens192", "ens224"], nics, {}), ({}, ["ens192", "ens224"]))
pinned, unresolved = pin_interface_names(["ens192", "ens224"], nics, {"ens192": self.MAC})
self.assertEqual(pinned, {"ens192": (self.MAC, "vif")})
self.assertEqual(unresolved, ["ens224"])
def test_link_file(self):
text = link_file("ens192", self.MAC, "vif")
self.assertIn("[Match]\nMACAddress=00:0c:29:d4:bf:19\nDriver=vif\n", text)
self.assertIn("[Link]\nName=ens192\n", text)
self.assertNotIn("Driver=", link_file("ens192", self.MAC, ""))
class GuestTreeTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = make_tree(Path(self.tmp.name), {
"lib": ("symlink", "usr/lib"),
"sbin": ("symlink", "/usr/sbin"), # absolute symlink, must stay in the guest
"etc/os-release": 'NAME="Debian GNU/Linux"\nID=debian\nVERSION_ID="12"\n',
"usr/sbin/update-initramfs": "",
"boot/vmlinuz-6.1.0-18-amd64": "",
"boot/initrd.img-6.1.0-18-amd64": "",
"usr/lib/modules/6.1.0-18-amd64/modules.dep":
"kernel/drivers/block/xen-blkfront.ko.xz: \nkernel/drivers/net/xen-netfront.ko.xz:\n",
"usr/lib/modules/6.1.0-18-amd64/modules.builtin":
"kernel/drivers/ata/ata_piix.ko\nkernel/drivers/scsi/sd_mod.ko\n",
"usr/lib/modules/5.10.0-old/modules.dep": "kernel/drivers/ata/ata_piix.ko:\n",
"usr/lib/modules/6.2.0-headers-only/README": "",
"etc/netplan/50-cloud-init.yaml": "network:\n ethernets:\n ens192:\n dhcp4: true\n",
"etc/sysconfig/network-scripts/ifcfg-ens224": "DEVICE=ens224\n",
"etc/selinux/config": "# comment\nSELINUX=enforcing\n",
})
def tearDown(self):
self.tmp.cleanup()
def test_resolve_absolute_symlink_stays_in_root(self):
self.assertEqual(resolve_in_root(self.root, "/sbin/update-initramfs"),
self.root / "usr/sbin/update-initramfs")
self.assertEqual(resolve_in_root(self.root, "/lib/modules"), self.root / "usr/lib/modules")
def test_os_release(self):
info = parse_os_release(guest_read(self.root, "/etc/os-release"))
self.assertEqual(info["ID"], "debian")
self.assertEqual(info["VERSION_ID"], "12")
def test_initramfs_tool(self):
self.assertEqual(detect_initramfs_tool(self.root), "initramfs-tools")
def test_kernels(self):
# 5.10.0-old has no vmlinuz, headers-only has no modules.dep
self.assertEqual(kernel_versions(self.root), ["6.1.0-18-amd64"])
self.assertEqual(initramfs_image(self.root, "6.1.0-18-amd64"), "/boot/initrd.img-6.1.0-18-amd64")
def test_available_modules(self):
self.assertEqual(available_modules(self.root, "6.1.0-18-amd64"),
["xen-blkfront", "xen-netfront", "ata_piix", "sd_mod"])
self.assertEqual(available_modules(self.root, "5.10.0-old"), ["ata_piix"])
def test_interface_names(self):
self.assertEqual(interface_names(self.root), ["ens192", "ens224"])
def test_selinux(self):
self.assertTrue(selinux_enabled(self.root))
def test_kdump_initrds(self):
self.assertEqual(stale_kdump_initrds(self.root), [])
self.assertEqual(rhel_kdump_images(self.root), [])
make_tree(self.root, {
"var/lib/kdump/initrd": "",
"boot/initramfs-5.14.0-427.el9.x86_64kdump.img": "",
"boot/initramfs-5.14.0-427.el9.x86_64.img": "",
})
self.assertEqual(stale_kdump_initrds(self.root), ["/var/lib/kdump/initrd"])
self.assertEqual(rhel_kdump_images(self.root), ["/boot/initramfs-5.14.0-427.el9.x86_64kdump.img"])
def test_disable_units(self):
make_tree(self.root, {
"lib/systemd/system/open-vm-tools.service": "",
"etc/systemd/system/multi-user.target.wants/open-vm-tools.service":
("symlink", "/lib/systemd/system/open-vm-tools.service"),
"etc/systemd/system/vmtoolsd.service": ("symlink", "/lib/systemd/system/open-vm-tools.service"),
"etc/systemd/system/vgauth.service": ("symlink", "/dev/null"), # masked: keep
"etc/systemd/system/multi-user.target.wants/ssh.service": ("symlink", "/lib/systemd/system/ssh.service"),
})
removed = disable_units(self.root, ("open-vm-tools.service", "vmtoolsd.service", "vgauth.service"))
self.assertEqual(removed, ["multi-user.target.wants/open-vm-tools.service", "vmtoolsd.service"])
etc = self.root / "etc/systemd/system"
self.assertTrue((etc / "vgauth.service").is_symlink())
self.assertTrue((etc / "multi-user.target.wants/ssh.service").is_symlink())
def test_disable_units_suse(self):
# openSUSE Leap 15.6: vgauthd is enabled on its own and required by vmtoolsd
make_tree(self.root, {
"usr/lib/systemd/system/vmtoolsd.service": "",
"usr/lib/systemd/system/vgauthd.service": "",
"etc/systemd/system/multi-user.target.wants/vmtoolsd.service":
("symlink", "/usr/lib/systemd/system/vmtoolsd.service"),
"etc/systemd/system/multi-user.target.wants/vgauthd.service":
("symlink", "/usr/lib/systemd/system/vgauthd.service"),
"etc/systemd/system/vmtoolsd.service.requires/vgauthd.service":
("symlink", "/usr/lib/systemd/system/vgauthd.service"),
"etc/systemd/system/multi-user.target.wants/ssh.service": ("symlink", "/lib/systemd/system/ssh.service"),
})
removed = disable_units(self.root, VMWARE_UNITS)
self.assertEqual(removed, [
"multi-user.target.wants/vgauthd.service", "multi-user.target.wants/vmtoolsd.service",
"vmtoolsd.service.requires/vgauthd.service",
])
etc = self.root / "etc/systemd/system"
self.assertTrue((etc / "multi-user.target.wants/ssh.service").is_symlink())
# Linux/ directory of guest-tools-8.3-19.xcpng8.3.iso
TOOLS_LINUX = [
"install.sh", "versions.deb", "versions.rpm", "xe-daemon",
"xe-guest-utilities_7.30.0-19_386.tgz", "xe-guest-utilities_7.30.0-19_amd64.deb",
"xe-guest-utilities_7.30.0-19_i386.deb", "xe-guest-utilities-7.30.0-19.i386.rpm",
"xe-guest-utilities-7.30.0-19.legacy.i386.rpm", "xe-guest-utilities-7.30.0-19.legacy.x86_64.rpm",
"xe-guest-utilities-7.30.0-19.x86_64.rpm", "xe-guest-utilities_7.30.0-19_x86_64.tgz",
"xe-linux-distribution", "xe-linux-distribution.service", "xen-vcpu-hotplug.rules",
]
# ELF header of an x86_64 binary, padded to the e_machine field
ELF_X86_64 = b"\x7fELF\x02\x01\x01" + b"\x00" * 11 + b"\x3e\x00"
ELF_I386 = b"\x7fELF\x01\x01\x01" + b"\x00" * 11 + b"\x03\x00"
ELF_AARCH64 = b"\x7fELF\x02\x01\x01" + b"\x00" * 11 + b"\xb7\x00"
class GuestAgentTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
def tearDown(self):
self.tmp.cleanup()
def test_select_package(self):
self.assertEqual(select_agent_package(TOOLS_LINUX, "rpm", "x86_64", True),
"xe-guest-utilities-7.30.0-19.x86_64.rpm")
self.assertEqual(select_agent_package(TOOLS_LINUX, "rpm", "x86_64", False),
"xe-guest-utilities-7.30.0-19.legacy.x86_64.rpm")
self.assertEqual(select_agent_package(TOOLS_LINUX, "dpkg", "x86_64", True),
"xe-guest-utilities_7.30.0-19_amd64.deb")
self.assertEqual(select_agent_package(TOOLS_LINUX, "dpkg", "i386", True),
"xe-guest-utilities_7.30.0-19_i386.deb")
self.assertIsNone(select_agent_package(TOOLS_LINUX, "dpkg", "aarch64", True))
self.assertEqual(select_agent_package(TOOLS_LINUX, "tgz", "x86_64", True),
"xe-guest-utilities_7.30.0-19_x86_64.tgz")
self.assertIsNone(select_agent_package(TOOLS_LINUX, "tgz", "x86_64", False)) # needs systemd
self.assertIsNone(select_agent_package(
[n for n in TOOLS_LINUX if not n.endswith(".service")], "tgz", "x86_64", True))
self.assertIsNone(select_agent_package(["install.sh"], "rpm", "x86_64", True))
def test_guest_arch_follows_symlinks(self):
make_tree(self.root, {"usr/bin/bash": ELF_X86_64, "bin": ("symlink", "usr/bin"),
"usr/bin/sh": ("symlink", "/usr/bin/bash")})
self.assertEqual(guest_arch(self.root), "x86_64")
def test_guest_arch_other(self):
make_tree(self.root, {"bin/sh": ELF_I386})
self.assertEqual(guest_arch(self.root), "i386")
(self.root / "bin/sh").write_bytes(ELF_AARCH64)
self.assertIsNone(guest_arch(self.root))
(self.root / "bin/sh").write_text("#!/bin/busybox\n")
self.assertIsNone(guest_arch(self.root))
def test_package_manager(self):
self.assertIsNone(package_manager(self.root))
make_tree(self.root, {"usr/bin/rpm": ""})
self.assertEqual(package_manager(self.root), "rpm")
make_tree(self.root, {"usr/bin/dpkg": ""}) # Debian with the rpm tool
self.assertEqual(package_manager(self.root), "dpkg")
def test_agent_enabled(self):
make_tree(self.root, {"etc/hostname": ""})
self.assertFalse(agent_enabled(self.root))
make_tree(self.root, {"etc/rc2.d/S01xe-linux-distribution": ("symlink", "../init.d/xe-linux-distribution")})
self.assertTrue(agent_enabled(self.root))
def test_agent_enabled_systemd(self):
make_tree(self.root, {"etc/systemd/system/multi-user.target.wants/xe-linux-distribution.service":
("symlink", "/usr/lib/systemd/system/xe-linux-distribution.service")})
self.assertTrue(agent_enabled(self.root))
def test_services_not_started_restores_existing_policy(self):
make_tree(self.root, {"usr/sbin/policy-rc.d": "#!/bin/sh\nexit 0\n"})
with services_not_started(self.root):
self.assertIn("exit 101", (self.root / "usr/sbin/policy-rc.d").read_text())
self.assertEqual((self.root / "usr/sbin/policy-rc.d").read_text(), "#!/bin/sh\nexit 0\n")
self.assertFalse((self.root / "usr/sbin/policy-rc.d.xo-docking.bak").exists())
def test_services_not_started_removes_its_policy(self):
make_tree(self.root, {"usr/sbin/dpkg": ""})
with self.assertRaises(RuntimeError):
with services_not_started(self.root):
raise RuntimeError("dpkg failed")
self.assertFalse((self.root / "usr/sbin/policy-rc.d").exists())
class FakeMounts:
"""Mounts stand-in: the guest root is a prepared tree, other mounts are no-ops."""
def __init__(self, root: Path):
self.root = root
def mount(self, device, fstype, target=None, *, readonly=True, options=()):
return Path(target) if target else self.root
def bind(self, source, target):
return target
class LeapGuestTest(unittest.TestCase):
"""openSUSE Leap 15.6 / SLES 15 installed with YaST "mount by device ID"."""
BY_ID = "/dev/disk/by-id/scsi-36000c29abc"
KVER = "6.4.0-150600.21-default"
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.tools = make_tree(Path(self.tmp.name) / "tools", {f"Linux/{n}": n for n in TOOLS_LINUX})
self.root = make_tree(Path(self.tmp.name) / "root", {
"bin": ("symlink", "usr/bin"),
"usr/bin/bash": ELF_X86_64,
"usr/bin/sh": ("symlink", "bash"),
"usr/bin/rpm": "",
"usr/lib/systemd/systemd": "",
"etc/os-release": 'NAME="openSUSE Leap"\nID="opensuse-leap"\nVERSION_ID="15.6"\n',
"etc/fstab": (
f"{self.BY_ID}-part2 / btrfs defaults 0 0\n"
f"{self.BY_ID}-part2 /var btrfs subvol=/@/var 0 0\n"
f"{self.BY_ID}-part3 swap swap defaults 0 0\n"
),
"etc/default/grub": f'GRUB_CMDLINE_LINUX_DEFAULT="splash=silent resume={self.BY_ID}-part3 quiet"\n',
"usr/bin/dracut": "",
f"boot/vmlinuz-{self.KVER}": "",
f"boot/initrd-{self.KVER}": "",
f"lib/modules/{self.KVER}/modules.dep": "kernel/drivers/block/xen-blkfront.ko.zst:\n",
"var/lib/kdump/initrd": "",
"usr/lib/systemd/system/vmtoolsd.service": "",
"usr/lib/systemd/system/vgauthd.service": "",
})
devices = parse_lsblk(lsblk_json(dev("xvda", "disk", children=[
dev("xvda1"), dev("xvda2", fstype="btrfs", uuid="root-uuid"), dev("xvda3", fstype="swap", uuid="swap-uuid"),
])))
root_dev = next(d for d in devices if d.name == "xvda2")
self.report = Report("dock", "test")
self.guest = LinuxGuest(OsCandidate("linux", root_dev), devices, guest_disks(devices),
FakeMounts(self.root), self.report, readonly=False, tools=self.tools)
self.guest.inspect()
def tearDown(self):
self.tmp.cleanup()
def action(self, code):
return next(a for a in self.report.actions if a.code == code)
def test_planned(self):
codes = [a.code for a in self.report.actions]
self.assertEqual(codes, ["fstab_stable_names", "disable_vmware_tools", "initramfs_xen_drivers",
"kdump_initrd_rebuild", "install_guest_agent"])
self.assertEqual(self.report.os["guest_agent"], "xe-guest-utilities-7.30.0-19.x86_64.rpm")
self.assertIn("vgauthd.service", self.action("disable_vmware_tools").description)
findings = {f.code for f in self.report.findings}
self.assertIn("grub_unstable_resume", findings)
self.assertNotIn("fstab_unstable_names", findings)
self.assertFalse(self.report.blocked) # /var found through the by-id name of /
def test_apply_fstab_and_kdump(self):
self.action("fstab_stable_names").apply()
self.action("kdump_initrd_rebuild").apply()
fstab = (self.root / "etc/fstab").read_text().splitlines()
self.assertEqual([line.split()[0] for line in fstab], ["UUID=root-uuid", "UUID=root-uuid", "UUID=swap-uuid"])
self.assertTrue((self.root / "etc/fstab.xo-docking.bak").is_file())
self.assertFalse((self.root / "var/lib/kdump/initrd").exists())
def test_apply_guest_agent(self):
for directory in ("dev/pts", "proc", "sys"):
(self.root / directory).mkdir(parents=True)
commands = []
def chroot_run(root, cmd, **kwargs):
commands.append(cmd)
staged = self.root / cmd[-1].lstrip("/")
self.assertTrue(staged.is_file()) # the package is in the guest when rpm runs
make_tree(self.root, {"etc/systemd/system/multi-user.target.wants/xe-linux-distribution.service":
("symlink", "/usr/lib/systemd/system/xe-linux-distribution.service")})
with mock.patch("xo_docking.linux.chroot_run", chroot_run):
self.action("install_guest_agent").apply()
self.assertEqual(commands, [["rpm", "-U", "--replacepkgs", "/tmp/xo-docking/xe-guest-utilities-7.30.0-19.x86_64.rpm"]])
self.assertFalse((self.root / "tmp/xo-docking").exists())
def test_apply_guest_agent_not_enabled(self):
for directory in ("dev/pts", "proc", "sys"):
(self.root / directory).mkdir(parents=True)
with mock.patch("xo_docking.linux.chroot_run"):
with self.assertRaises(DockingError):
self.action("install_guest_agent").apply()
class ReadonlyRootTest(unittest.TestCase):
"""SL Micro: read-only btrfs root snapshot, managed with transactional-update."""
KVER = "6.12.0-160000.5-default"
MODULES = f"lib/modules/{KVER}"
def plan(self, initramfs_listing):
with tempfile.TemporaryDirectory() as tmp:
root = make_tree(Path(tmp) / "root", {
"etc/os-release": 'NAME="SLES"\nVARIANT="Micro"\nID="sles"\nVERSION_ID="16.0"\n',
"etc/fstab": "UUID=1ef4 / btrfs ro 0 0\nUUID=EB0F /boot/efi vfat defaults 0 0\n",
"usr/sbin/transactional-update": "",
"usr/bin/dracut": "",
"usr/bin/rpm": "",
"usr/lib/systemd/systemd": "",
"bin/sh": ELF_X86_64,
f"boot/vmlinuz-{self.KVER}": "",
f"boot/initrd-{self.KVER}": "",
f"{self.MODULES}/modules.dep": "kernel/drivers/block/xen-blkfront.ko.zst:\n"
"kernel/drivers/net/xen-netfront.ko.zst:\n",
"usr/lib/systemd/system/vmtoolsd.service": "",
})
tools_dir = make_tree(Path(tmp) / "tools", {f"Linux/{n}": "" for n in TOOLS_LINUX})
devices = parse_lsblk(lsblk_json(dev("xvda", "disk", children=[dev("xvda3", fstype="ext4")])))
report = Report("dock", "test")
guest = LinuxGuest(OsCandidate("linux", devices[1]), devices, guest_disks(devices),
FakeMounts(root), report, readonly=False, tools=tools_dir)
with mock.patch("xo_docking.linux.initramfs_files", return_value=initramfs_listing):
guest.inspect()
guest.finalize()
return report
def test_initramfs_already_has_the_drivers(self):
report = self.plan([f"usr/lib/modules/{self.KVER}/kernel/drivers/block/xen-blkfront.ko.zst",
f"usr/lib/modules/{self.KVER}/kernel/drivers/net/xen-netfront.ko.zst"])
self.assertEqual(report.actions, []) # nothing is written offline
codes = {f.code: f for f in report.findings}
self.assertIn("initramfs_xen_drivers_present", codes)
self.assertEqual(codes["readonly_root"].data["actions"], ["disable_vmware_tools", "install_guest_agent"])
self.assertFalse(report.blocked)
self.assertTrue(report.os["readonly_root"])
def test_initramfs_without_the_drivers_is_blocking(self):
report = self.plan([f"usr/lib/modules/{self.KVER}/kernel/drivers/net/xen-netfront.ko.zst"])
errors = [f for f in report.findings if f.level == "error"]
self.assertEqual([f.code for f in errors], ["readonly_root_initramfs"])
self.assertEqual(errors[0].data["kernels"], {self.KVER: ["xen-blkfront"]})
self.assertTrue(report.blocked)
def test_missing_initramfs_modules(self):
files = ["usr/lib/modules/6.1/kernel/drivers/block/xen-blkfront.ko.xz", "usr/bin/sh"]
self.assertEqual(missing_initramfs_modules(files, ["xen-blkfront", "xen-netfront"]), ["xen-netfront"])
self.assertEqual(missing_initramfs_modules([], ["xen_blkfront"]), ["xen_blkfront"])
class GuestAgentPlanTest(unittest.TestCase):
def plan(self, files, tools):
with tempfile.TemporaryDirectory() as tmp:
root = make_tree(Path(tmp) / "root", {"etc/os-release": "ID=arch\n", "etc/fstab": "", **files})
tools_dir = make_tree(Path(tmp) / "tools", {f"Linux/{n}": "" for n in TOOLS_LINUX}) if tools else None
devices = parse_lsblk(lsblk_json(dev("xvda", "disk", children=[dev("xvda1", fstype="ext4")])))
report = Report("diagnose", "test")
guest = LinuxGuest(OsCandidate("linux", devices[1]), devices, guest_disks(devices),
FakeMounts(root), report, readonly=True, tools=tools_dir)
guest.inspect()
return report
def codes(self, report):
return {f.code for f in report.findings} | {a.code for a in report.actions}
def test_already_installed(self):
codes = self.codes(self.plan({"usr/sbin/xe-daemon": ""}, tools=True))
self.assertFalse(codes & {"install_guest_agent", "guest_agent_missing", "guest_agent_unsupported"})
def test_no_tools_disk(self):
self.assertIn("guest_agent_missing", self.codes(self.plan({}, tools=False)))
def test_rhel_kdump_image_deleted(self):
report = self.plan({"boot/initramfs-3.10.0-514.el7.x86_64kdump.img": ""}, tools=False)
action = next(a for a in report.actions if a.code == "kdump_initrd_rebuild")
self.assertIn("/boot/initramfs-3.10.0-514.el7.x86_64kdump.img", action.description)
self.assertFalse(action.critical)
def test_no_package_manager_nor_systemd(self):
report = self.plan({"bin/sh": ELF_X86_64, "usr/bin/pacman": ""}, tools=True)
self.assertIn("guest_agent_unsupported", self.codes(report))
self.assertNotIn("install_guest_agent", self.codes(report))
def test_no_package_manager_with_systemd(self): # Arch Linux: generic tarball
report = self.plan({"bin/sh": ELF_X86_64, "usr/lib/systemd/systemd": ""}, tools=True)
self.assertIn("install_guest_agent", self.codes(report))
self.assertEqual(report.os["guest_agent"], "xe-guest-utilities_7.30.0-19_x86_64.tgz")
class AgentTarballTest(unittest.TestCase):
def test_extract_follows_merged_usr_and_enables_unit(self):
with tempfile.TemporaryDirectory() as tmp:
root = make_tree(Path(tmp) / "root", {"usr/bin/bash": ELF_X86_64, "usr/sbin": ("symlink", "bin"),
"etc/hostname": ""})
tgz = Path(tmp) / "agent.tgz"
with tarfile.open(tgz, "w:gz") as archive:
for name, data, mode in (("usr/sbin/xe-daemon", b"ELF", 0o755),
("etc/init.d/xe-linux-distribution", b"#!/bin/sh", 0o755),
("../escape", b"x", 0o644)):
info = tarfile.TarInfo(name)
info.size, info.mode = len(data), mode
archive.addfile(info, io.BytesIO(data))
link = tarfile.TarInfo("usr/bin/xenstore-read")
link.type, link.linkname = tarfile.SYMTYPE, "xenstore"
archive.addfile(link)
installed = extract_agent_tgz(root, tgz)
self.assertEqual(installed, ["/usr/sbin/xe-daemon", "/usr/bin/xenstore-read"])
self.assertTrue((root / "usr/bin/xe-daemon").is_file()) # through the /usr/sbin link
self.assertTrue(os.access(root / "usr/bin/xe-daemon", os.X_OK))
self.assertFalse((root / "etc/init.d").exists())
self.assertFalse((Path(tmp) / "escape").exists())
self.assertEqual(os.readlink(root / "usr/bin/xenstore-read"), "xenstore")
enable_unit(root, "xe-linux-distribution.service")
self.assertTrue(agent_enabled(root))
if __name__ == "__main__":
unittest.main()

View File

@@ -1,152 +0,0 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest import mock
import helpers # noqa: F401 # sets the import path
from xo_docking.mounts import Mounts, is_readonly_mount, mount_options
from xo_docking.util import DockingError
from xo_docking.report import Report
class ReportTest(unittest.TestCase):
def test_result(self):
report = Report("diagnose", "dev")
self.assertEqual(report.result(), "ok")
report.warning("w", "warning")
self.assertEqual(report.result(), "warning")
report.error("e", "error")
self.assertEqual(report.result(), "error")
self.assertTrue(report.blocked)
def test_critical_failure_stops(self):
report = Report("dock", "dev")
applied = []
def boom():
raise RuntimeError("boom")
report.plan("first", "ok", lambda: applied.append("first"))
report.plan("optional", "fails", boom, critical=False)
report.plan("critical", "fails", boom)
report.plan("last", "never", lambda: applied.append("last"))
report.apply_actions()
self.assertEqual(applied, ["first"])
self.assertEqual([a.status for a in report.actions], ["applied", "failed", "failed", "skipped"])
self.assertEqual(report.result(), "error")
def test_compact_json_fits_and_keeps_errors(self):
report = Report("diagnose", "20261011-abcdef12")
for i in range(200):
report.info(f"info_{i}", "x" * 100)
report.warning(f"warn_{i}", "y" * 100, details=list(range(20)))
report.error("blocking", "the one that matters")
text = report.compact_json()
self.assertLessEqual(len(text.encode()), 1900)
data = json.loads(text)
self.assertTrue(data["truncated"])
self.assertIn("blocking", [f["code"] for f in data["findings"]])
self.assertEqual(data["result"], "error")
def test_compact_json_drops_what_xo_does_not_read_first(self):
report = Report("diagnose", "dev")
report.os = {"kind": "linux", "name": "Debian 12", "kernels": [f"6.1.0-{i}-amd64" for i in range(100)]}
report.error("boot_disk_not_first", "the OS is not on the first disk", os_disk="xvdb")
data = json.loads(report.compact_json())
self.assertEqual(data["os"], {"kind": "linux", "name": "Debian 12"})
# the details XO reads in a finding are kept longer than those of the OS
self.assertEqual(data["findings"][0]["data"], {"os_disk": "xvdb"})
self.assertNotIn("truncated", data)
def test_compact_json_keeps_what_xo_relies_on(self):
report = Report("dock", "dev")
report.os = {"kind": "windows", "rtc_timeoffset": 3600, "vmware_services": [f"vm{i}" for i in range(100)]}
report.network = [
{"adapter": "vmxnet3 Ethernet Adapter " * 4, "guid": f"{{guid-{i}}}", "dhcp": i % 2 == 0,
"addresses": [f"10.0.0.{i}"], "masks": ["255.255.255.0"], "gateways": ["10.0.0.1"],
"dns": ["10.0.0.2", "10.0.0.3"]}
for i in range(10)
]
report.warning("pv_drivers_unsupported", "too old")
for i in range(100):
report.warning(f"warn_{i}", "y" * 100)
report.plan("install_firstboot", "first boot", lambda: None)
for i in range(100):
report.plan(f"action_{i}", "x" * 100, lambda: None)
report.apply_actions()
text = report.compact_json()
self.assertLessEqual(len(text.encode()), 1900)
data = json.loads(text)
self.assertTrue(data["truncated"])
self.assertEqual(data["os"], {"kind": "windows", "rtc_timeoffset": 3600})
self.assertEqual(data["network"], [
{"guid": f"{{guid-{i}}}", "dhcp": False, "addresses": [f"10.0.0.{i}"]} for i in range(1, 10, 2)
])
self.assertIn("pv_drivers_unsupported", [f["code"] for f in data["findings"]])
self.assertIn({"code": "install_firstboot", "status": "applied"}, data["actions"])
def test_compact_json_keeps_the_info_xo_acts_on(self):
report = Report("dock", "dev")
report.info("guest_agent_missing", "no guest tools disk")
for i in range(100):
report.info(f"info_{i}", "x" * 100)
data = json.loads(report.compact_json())
self.assertEqual([f["code"] for f in data["findings"]], ["guest_agent_missing"])
def test_compact_json_small_report_is_complete(self):
report = Report("dock", "dev")
report.warning("windows_static_ip", "static IP", addresses=["10.0.0.5"])
data = json.loads(report.compact_json())
self.assertEqual(data["findings"][0]["data"], {"addresses": ["10.0.0.5"]})
self.assertNotIn("truncated", data)
class MountOptionsTest(unittest.TestCase):
def test_is_readonly_mount(self):
with tempfile.TemporaryDirectory() as tmp:
mounts = Path(tmp) / "mounts"
mounts.write_text("/dev/xvda3 /run/xo-docking/mnt/m1 fuseblk ro,relatime,user_id=0 0 0\n"
"/dev/xvda2 /run/xo-docking/mnt/m2 fuseblk rw,relatime,user_id=0 0 0\n")
self.assertTrue(is_readonly_mount(Path("/run/xo-docking/mnt/m1"), str(mounts)))
self.assertFalse(is_readonly_mount(Path("/run/xo-docking/mnt/m2"), str(mounts)))
self.assertFalse(is_readonly_mount(Path("/run/xo-docking/mnt/m3"), str(mounts)))
def test_ntfs_read_only_fallback_is_an_error(self):
"""ntfs-3g mounts an unclean volume read-only, with success."""
with tempfile.TemporaryDirectory() as tmp:
mounts = Mounts(Path(tmp))
ok = mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch("xo_docking.mounts.run", return_value=ok), \
mock.patch("xo_docking.mounts.is_readonly_mount", return_value=True):
with self.assertRaises(DockingError) as caught:
mounts.mount("/dev/xvda2", "ntfs", readonly=False)
self.assertEqual(caught.exception.code, "ntfs_unclean")
self.assertEqual(mounts._stack, []) # unmounted again
def test_readonly_never_replays_journals(self):
self.assertEqual(mount_options("ext4", True), ["ro", "noload"])
self.assertEqual(mount_options("xfs", True), ["ro", "norecovery"])
self.assertEqual(mount_options("btrfs", True, ["subvol=@"]), ["ro", "rescue=nologreplay", "subvol=@"])
self.assertEqual(mount_options("ntfs", True), ["ro"])
def test_readwrite(self):
self.assertEqual(mount_options("ext4", False), ["rw"])
self.assertEqual(mount_options("ntfs", False), [])
class MountsTest(unittest.TestCase):
def test_mounted_reuses_existing_mount(self):
ok = mock.Mock(returncode=0)
with tempfile.TemporaryDirectory() as tmp, mock.patch("xo_docking.mounts.run", return_value=ok):
mounts = Mounts(tmp)
target = mounts.mount("/dev/xvda1", "vfat", Path(tmp) / "boot-efi")
self.assertEqual(mounts.mounted("/dev/xvda1"), target)
self.assertIsNone(mounts.mounted("/dev/xvda2"))
mounts.close()
self.assertIsNone(mounts.mounted("/dev/xvda1"))
if __name__ == "__main__":
unittest.main()

View File

@@ -1,60 +0,0 @@
import tempfile
import unittest
from pathlib import Path
from helpers import dev, lsblk_json, make_tree
from xo_docking.blockdev import parse_lsblk
from xo_docking.report import Report
from xo_docking.tools import find_tools
class FakeMounts:
"""Each device "mounts" on a prepared directory."""
def __init__(self, trees: dict):
self.trees = trees
self.unmounted = []
def mount(self, device, fstype, target=None, *, readonly=True, options=()):
assert readonly
return self.trees[device]
def umount(self, target):
self.unmounted.append(target)
class FindToolsTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
base = Path(self.tmp.name)
self.trees = {
"/dev/xvdy": make_tree(base / "other", {"README": ""}),
"/dev/xvdz": make_tree(base / "tools", {"Linux/install.sh": "", "Windows/XenTools-x64.msi": ""}),
}
self.devices = parse_lsblk(lsblk_json(
dev("xvda", "disk", children=[dev("xvda1", fstype="ext4")]),
dev("xvdy", "disk", fstype="iso9660"),
dev("xvdz", "disk", fstype="iso9660"),
))
def tearDown(self):
self.tmp.cleanup()
def test_found(self):
mounts = FakeMounts(self.trees)
report = Report("diagnose", "test")
self.assertEqual(find_tools(self.devices, mounts, report), self.trees["/dev/xvdz"])
self.assertEqual(report.tools, "xvdz")
self.assertEqual(mounts.unmounted, [self.trees["/dev/xvdy"]]) # not the guest tools
self.assertEqual(report.to_dict()["tools"], "xvdz")
def test_absent(self):
report = Report("diagnose", "test")
devices = [d for d in self.devices if d.name != "xvdz"]
self.assertIsNone(find_tools(devices, FakeMounts(self.trees), report))
self.assertIsNone(report.tools)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,334 +0,0 @@
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from helpers import make_tree, write_empty_hive
from xo_docking.efi import fallback_copies
from xo_docking.windows import (
FIRSTBOOT_SERVICE, clear_static_config, control_sets, current_control_set, disable_services,
enable_boot_drivers, firstboot_config, firstboot_msi_patterns, firstboot_state, install_firstboot_service, is_hibernated, match_packages,
map_static_adapters, network_adapters, network_config, parse_windows_nics, pv_drivers_installed,
rtc_timeoffset, vmware_services, WindowsGuest,
)
try:
from xo_docking.hive import Hive
import hivex # noqa: F401 # pyflakes: availability check
HAVE_HIVEX = True
except ImportError:
HAVE_HIVEX = False
NET_CLASS = "{4d36e972-e325-11ce-bfc1-08002be10318}"
OLD_NIC = "{11111111-2222-3333-4444-555555555555}"
LOOPBACK = "{99999999-2222-3333-4444-555555555555}"
def build_control_set(hive, root, name):
control_set = hive.add_child(root, name)
services = hive.add_child(control_set, "Services")
def service(service_name, start, **values):
node = hive.add_child(services, service_name)
hive.set_dword(node, "Start", start)
for key, value in values.items():
if isinstance(value, int):
hive.set_dword(node, key, value)
else:
hive.set_string(node, key, value, expand=key == "ImagePath")
return node
intelide = service("intelide", 3)
hive.add_child(intelide, "StartOverride")
service("atapi", 3)
service("VMTools", 2, ImagePath=r'"C:\Program Files\VMware\VMware Tools\vmtoolsd.exe"', Type=0x10)
service("vmci", 0, Type=1, ImagePath=r"\SystemRoot\System32\drivers\vmci.sys")
# unknown to the static list, found by the generic rule
service("vmgencounter", 3, Type=1, DisplayName="VMware Generation Counter",
ImagePath=r"\SystemRoot\System32\drivers\vmgencounter.sys")
service("VMwareToolsNewThing", 2, Type=0x10,
ImagePath=r"C:\Program Files\VMware\VMware Tools\new.exe")
# must NOT be caught: security agent from the VMware/Broadcom catalog
service("CbDefense", 2, Type=0x10, DisplayName="VMware Carbon Black Cloud Sensor",
ImagePath=r"C:\Program Files\Confer\RepMgr.exe")
service("vmdisabled", 4, Type=1, DisplayName="VMware already disabled")
tz = hive.add_child(hive.add_child(control_set, "Control"), "TimeZoneInformation")
hive.set_dword(tz, "Bias", 0xFFFFFFC4) # -60: UTC+1
hive.set_dword(tz, "ActiveTimeBias", 0xFFFFFF88) # -120: UTC+2, summer time
interfaces = hive.add_child(hive.add_child(hive.add_child(services, "Tcpip"), "Parameters"), "Interfaces")
old = hive.add_child(interfaces, OLD_NIC)
hive.set_dword(old, "EnableDHCP", 0)
hive.set_strings(old, "IPAddress", ["192.168.1.10"])
hive.set_strings(old, "SubnetMask", ["255.255.255.0"])
hive.set_strings(old, "DefaultGateway", ["192.168.1.1"])
hive.set_string(old, "NameServer", "192.168.1.53,192.168.1.54")
loop = hive.add_child(interfaces, LOOPBACK)
hive.set_dword(loop, "EnableDHCP", 0)
hive.set_strings(loop, "IPAddress", ["10.9.9.9"])
adapters = hive.add_child(hive.add_child(hive.add_child(control_set, "Control"), "Class"), NET_CLASS)
nic = hive.add_child(adapters, "0007")
hive.set_string(nic, "ComponentId", "PCI\\VEN_15AD&DEV_07B0&SUBSYS_07B015AD")
hive.set_string(nic, "NetCfgInstanceId", OLD_NIC)
hive.set_string(nic, "DriverDesc", "vmxnet3 Ethernet Adapter")
software_nic = hive.add_child(adapters, "0008")
hive.set_string(software_nic, "ComponentId", "*msloop")
hive.set_string(software_nic, "NetCfgInstanceId", LOOPBACK)
return control_set
@unittest.skipUnless(HAVE_HIVEX, "python3-hivex is not installed")
class SystemHiveTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.path = write_empty_hive(Path(self.tmp.name) / "SYSTEM")
hive = Hive(self.path, write=True)
root = hive.root()
hive.set_dword(hive.add_child(root, "Select"), "Current", 2)
build_control_set(hive, root, "ControlSet001") # Last Known Good
build_control_set(hive, root, "ControlSet002") # current
hive.add_child(root, "MountedDevices")
hive.commit()
hive.close()
def tearDown(self):
self.tmp.cleanup()
def test_detection(self):
hive = Hive(self.path)
current = current_control_set(hive)
self.assertEqual(hive._h.node_name(current), "ControlSet002")
self.assertEqual(len(control_sets(hive)), 2)
services = hive.child(current, "Services")
self.assertEqual(vmware_services(hive, services),
["vmci", "vmgencounter", "VMTools", "VMwareToolsNewThing"])
adapters = network_adapters(hive, current)
self.assertEqual(adapters, [{
"adapter": "vmxnet3 Ethernet Adapter", "vmware": True, "guid": OLD_NIC, "dhcp": False,
"addresses": ["192.168.1.10"], "masks": ["255.255.255.0"], "gateways": ["192.168.1.1"],
"dns": ["192.168.1.53", "192.168.1.54"],
}])
self.assertEqual(rtc_timeoffset(hive, current), 7200)
hive.close()
def test_rtc_universal(self):
hive = Hive(self.path, write=True)
current = current_control_set(hive)
tz = hive.node("ControlSet002", "Control", "TimeZoneInformation")
hive.set_dword(tz, "RealTimeIsUniversal", 1)
self.assertEqual(rtc_timeoffset(hive, current), 0)
hive.close()
def test_dock_applies_to_every_control_set(self):
hive = Hive(self.path, write=True)
services_current = hive.child(current_control_set(hive), "Services")
names = vmware_services(hive, services_current)
for control_set in control_sets(hive):
services = hive.child(control_set, "Services")
enable_boot_drivers(hive, services)
disable_services(hive, services, names)
clear_static_config(hive, control_set, [OLD_NIC])
install_firstboot_service(hive, services)
hive.commit()
hive.close()
hive = Hive(self.path)
for control_set in control_sets(hive):
services = hive.child(control_set, "Services")
intelide = hive.child(services, "intelide")
self.assertEqual(hive.dword(intelide, "Start"), 0)
self.assertIsNone(hive.child(intelide, "StartOverride"))
for name in names:
self.assertEqual(hive.dword(hive.child(services, name), "Start"), 4, name)
self.assertEqual(hive.dword(hive.child(services, "CbDefense"), "Start"), 2)
firstboot = hive.child(services, FIRSTBOOT_SERVICE)
self.assertEqual(hive.dword(firstboot, "Start"), 2)
self.assertIn(r"\xo-docking\firstboot.ps1", hive.string(firstboot, "ImagePath"))
self.assertEqual(network_adapters(hive, control_set)[0]["addresses"], [])
self.assertTrue(network_adapters(hive, control_set)[0]["dhcp"])
hive.close()
def test_pv_drivers_installed(self):
hive = Hive(self.path, write=True)
services = hive.child(current_control_set(hive), "Services")
self.assertFalse(pv_drivers_installed(hive, services)) # fresh from VMware
for name in ("xenbus", "xenvbd"):
hive.set_dword(hive.add_child(services, name), "Start", 0)
self.assertTrue(pv_drivers_installed(hive, services))
hive.set_dword(hive.child(services, "xenvbd"), "Start", 4)
self.assertFalse(pv_drivers_installed(hive, services))
hive.close()
def test_firstboot_config(self):
self.assertEqual(firstboot_config(["xen-*.msi", "managementagent-*.msi"], "full"),
b"msi_patterns=xen-*.msi;managementagent-*.msi\r\ncleanup=full\r\nreboot=guest\r\nmax_attempts=3\r\n")
self.assertIn(b"\r\nreboot=xo\r\n", firstboot_config(["*.msi"], "standard", "xo"))
class FirstbootMsiPatternsTest(unittest.TestCase):
def test_default_depends_on_build(self):
self.assertEqual(firstboot_msi_patterns(20348, None), ["*.msi"])
self.assertEqual(firstboot_msi_patterns(14393, None), ["*.msi"])
# 9.x MSI unsupported: no package, so the first boot reaches the static IP phase
self.assertEqual(firstboot_msi_patterns(9600, None), [])
def test_given_patterns_always_used(self):
self.assertEqual(firstboot_msi_patterns(9600, ["old-tools*.msi"]), ["old-tools*.msi"])
self.assertEqual(firstboot_msi_patterns(20348, []), [])
def test_empty_config(self):
self.assertEqual(firstboot_config([], "standard"),
b"msi_patterns=\r\ncleanup=standard\r\nreboot=guest\r\nmax_attempts=3\r\n")
class StaticNetworkTest(unittest.TestCase):
OLD = {"adapter": "vmxnet3 Ethernet Adapter", "vmware": True, "guid": "{44FED271-EE81-4363-8CF0-893542F0734F}",
"dhcp": False, "addresses": ["192.168.1.246"], "masks": ["255.255.255.0"],
"gateways": ["192.168.1.254"], "dns": ["192.168.1.254", "1.1.1.1"]}
BACKEND = {"adapter": "vmxnet3 Ethernet Adapter #2", "vmware": True, "guid": "{55FED271-EE81-4363-8CF0-893542F0734F}",
"dhcp": False, "addresses": ["10.0.0.5"], "masks": ["255.255.0.0"], "gateways": [], "dns": []}
def test_parse_windows_nics(self):
value = "00:0C:29:94:80:0D={44FED271-EE81-4363-8CF0-893542F0734F};bad=x;00-0c-29-94-80-17={55FED271-EE81-4363-8CF0-893542F0734F}"
self.assertEqual(parse_windows_nics(value), {
"00:0c:29:94:80:0d": "{44fed271-ee81-4363-8cf0-893542f0734f}",
"00:0c:29:94:80:17": "{55fed271-ee81-4363-8cf0-893542f0734f}",
})
self.assertEqual(parse_windows_nics(None), {})
def test_single_adapter_is_automatic(self):
mapped, unmapped = map_static_adapters([self.OLD], ["00:0c:29:94:80:0d"], {})
self.assertEqual(mapped, [("00:0c:29:94:80:0d", self.OLD)])
self.assertEqual(unmapped, [])
def test_several_adapters_need_xo_mapping(self):
macs = ["00:0c:29:94:80:0d", "00:0c:29:94:80:17"]
mapped, unmapped = map_static_adapters([self.OLD, self.BACKEND], macs, {})
self.assertEqual((mapped, unmapped), ([], [self.OLD, self.BACKEND]))
given = parse_windows_nics("00:0c:29:94:80:17={55FED271-EE81-4363-8CF0-893542F0734F}")
mapped, unmapped = map_static_adapters([self.OLD, self.BACKEND], macs, given)
self.assertEqual(mapped, [("00:0c:29:94:80:17", self.BACKEND)])
self.assertEqual(unmapped, [self.OLD])
def test_network_config(self):
self.assertEqual(network_config([("00:0c:29:94:80:0d", self.OLD), ("00:0c:29:94:80:17", self.BACKEND)]),
b"00:0c:29:94:80:0d|192.168.1.246|255.255.255.0|192.168.1.254|192.168.1.254,1.1.1.1\r\n"
b"00:0c:29:94:80:17|10.0.0.5|255.255.0.0||\r\n")
class MatchPackagesTest(unittest.TestCase):
# guest-tools-8.3-19.xcpng8.3.iso
FILES = ["Linux/install.sh", "Windows/XenBootFix/XenBootFix.exe", "Windows/XenClean/XenClean.exe",
"Windows/XenTools-x64.msi", "Windows/XenTools-fix-9.2.351.msp"]
def test_default_pattern(self):
self.assertEqual(match_packages(self.FILES, ["*.msi"]), {"*.msi": "Windows/XenTools-x64.msi"})
def test_case_insensitive_on_the_name_only(self):
self.assertEqual(match_packages(self.FILES, ["xentools-*.MSI", "windows*"]),
{"xentools-*.MSI": "Windows/XenTools-x64.msi", "windows*": None})
def test_order_kept(self):
files = ["b/managementagent-x64.msi", "a/xen-drivers.msi"]
self.assertEqual(list(match_packages(files, ["xen-*.msi", "managementagent-*.msi"]).values()),
["a/xen-drivers.msi", "b/managementagent-x64.msi"])
class FirstbootStateTest(unittest.TestCase):
def state(self, files):
with tempfile.TemporaryDirectory() as tmp:
base = make_tree(Path(tmp), {"firstboot.ps1": "", "config.ini": "max_attempts = 3\n", **files})
return firstboot_state(base)
def test_not_installed(self):
with tempfile.TemporaryDirectory() as tmp:
self.assertIsNone(firstboot_state(Path(tmp)))
self.assertIsNone(firstboot_state(None))
def test_states(self):
self.assertEqual(self.state({})["status"], "drivers:pending")
self.assertEqual(self.state({"phase": "drivers\n", "attempts": "1\n"})["status"], "drivers:error")
self.assertEqual(self.state({"phase": "drivers", "attempts": "3"})["status"], "drivers:failed")
self.assertEqual(self.state({"phase": "finish", "attempts": "1", "firstboot.done": "x"})["status"], "done")
self.assertEqual(self.state({"phase": "done"})["status"], "done")
def test_log_tail(self):
log = "".join(f"line {i}\n" for i in range(10)) + "x" * 500 + "\n"
state = self.state({"attempts": "1", "firstboot.log": log})
self.assertEqual(state["log"][:4], ["line 6", "line 7", "line 8", "line 9"])
self.assertEqual(len(state["log"][4]), 200)
class HibernationTest(unittest.TestCase):
def test_signatures(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.assertFalse(is_hibernated(root))
(root / "hiberfil.sys").write_bytes(b"HIBR" + bytes(60))
self.assertTrue(is_hibernated(root))
(root / "hiberfil.sys").write_bytes(b"wake" + bytes(60))
self.assertFalse(is_hibernated(root))
(root / "hiberfil.sys").write_bytes(bytes(64))
self.assertFalse(is_hibernated(root))
class EfiFallbackTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.esp = Path(self.tmp.name)
def tearDown(self):
self.tmp.cleanup()
def names(self, copies):
return sorted((source.relative_to(self.esp).as_posix(), target) for source, target in copies.items())
def test_debian_without_removable_path(self):
make_tree(self.esp, {
"EFI/debian/shimx64.efi": "", "EFI/debian/grubx64.efi": "", "EFI/debian/mmx64.efi": "",
"EFI/debian/grub.cfg": "", "EFI/ubuntu/grubx64.efi": "",
})
self.assertEqual(self.names(fallback_copies(self.esp / "EFI", "linux", ["debian"])), [
("EFI/debian/grub.cfg", "grub.cfg"), ("EFI/debian/grubx64.efi", "grubx64.efi"),
("EFI/debian/mmx64.efi", "mmx64.efi"), ("EFI/debian/shimx64.efi", "BOOTX64.EFI"),
])
def test_rhel_vendor_alias(self):
make_tree(self.esp, {"EFI/centos/grubx64.efi": "", "EFI/redhat/shimx64.efi": ""})
copies = fallback_copies(self.esp / "EFI", "linux", ["rhel", "fedora"])
self.assertEqual(self.names(copies), [("EFI/redhat/shimx64.efi", "BOOTX64.EFI")])
def test_existing_fallback_case_insensitive(self):
make_tree(self.esp, {"EFI/Boot/bootx64.efi": "", "EFI/debian/shimx64.efi": ""})
self.assertEqual(fallback_copies(self.esp / "EFI", "linux", ["debian"]), {})
def test_windows(self):
make_tree(self.esp, {"EFI/Microsoft/Boot/bootmgfw.efi": ""})
self.assertEqual(self.names(fallback_copies(self.esp / "EFI", "windows", ["windows"])),
[("EFI/Microsoft/Boot/bootmgfw.efi", "BOOTX64.EFI")])
def test_no_loader(self):
make_tree(self.esp, {"EFI/empty/README": ""})
self.assertIsNone(fallback_copies(self.esp / "EFI", "linux", ["debian"]))
class CommitTest(unittest.TestCase):
def test_backup_keeps_the_hive_before_the_first_docking(self):
with tempfile.TemporaryDirectory() as tmp:
guest = WindowsGuest.__new__(WindowsGuest)
guest.system_path = Path(tmp) / "SYSTEM"
guest.system_path.write_bytes(b"original")
guest.system = mock.Mock()
guest._commit()
guest.system_path.write_bytes(b"docked") # what the first run wrote
guest._commit()
self.assertEqual((Path(tmp) / "SYSTEM.xo-docking.bak").read_bytes(), b"original")
if __name__ == "__main__":
unittest.main()

View File

@@ -1,75 +0,0 @@
import json
import os
import stat
import tempfile
import unittest
from pathlib import Path
from unittest import mock
import helpers # noqa: F401 # import path
from xo_docking.xenstore import STATUS_PREFIX, Xenstore
# Mimics the xenstore CLI: without -R, values are unescaped on write
# (\\ -> \, \xNN -> byte) and escaped on read.
FAKE = r'''#!/usr/bin/env python3
import json, re, sys
from pathlib import Path
store = Path(__file__).with_name("store.json")
data = json.loads(store.read_text()) if store.exists() else {}
cmd = Path(sys.argv[0]).name
args = sys.argv[1:]
raw = "-R" in args
args = [a for a in args if a != "-R"]
if cmd == "xenstore-write":
value = args[1] if raw else re.sub(r"\\(x[0-9a-fA-F]{2}|.)",
lambda m: chr(int(m.group(1)[1:], 16)) if m.group(1)[0] == "x" and len(m.group(1)) == 3 else m.group(1),
args[1])
data[args[0]] = value
elif cmd == "xenstore-read":
if args[0] not in data:
sys.exit(1)
print(data[args[0]] if raw else data[args[0]].replace("\\", "\\\\"))
elif cmd == "xenstore-rm":
data = {k: v for k, v in data.items() if not (k == args[0] or k.startswith(args[0] + "/"))}
store.write_text(json.dumps(data))
'''
class XenstoreTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
bindir = Path(self.tmp.name)
for name in ("xenstore-read", "xenstore-write", "xenstore-rm"):
path = bindir / name
path.write_text(FAKE)
path.chmod(path.stat().st_mode | stat.S_IXUSR)
self.store = bindir / "store.json"
env = mock.patch.dict(os.environ, {"PATH": f"{bindir}{os.pathsep}{os.environ['PATH']}"})
env.start()
self.addCleanup(env.stop)
self.xs = Xenstore(True)
def tearDown(self):
self.tmp.cleanup()
def test_report_with_backslashes_stays_valid_json(self):
report = json.dumps({"message": r"the ESP already has \EFI\BOOT\BOOTX64.EFI", "x": r"C:\x41"})
self.assertTrue(self.xs.status("report", report))
stored = json.loads(self.store.read_text())[f"{STATUS_PREFIX}/report"]
self.assertEqual(json.loads(stored)["message"], r"the ESP already has \EFI\BOOT\BOOTX64.EFI")
self.assertEqual(stored, report)
def test_config_with_backslash_read_raw(self):
self.xs.write("vm-data/xo-docking/windows-msi-patterns", r"drivers\xen-*.msi")
self.assertEqual(self.xs.config("windows-msi-patterns"), r"drivers\xen-*.msi")
def test_clear_status(self):
self.xs.status("state", "done")
self.xs.write("vm-data/xo-docking/mode", "dock")
self.xs.clear_status()
self.assertEqual(json.loads(self.store.read_text()), {"vm-data/xo-docking/mode": "dock"})
if __name__ == "__main__":
unittest.main()

View File

@@ -27,7 +27,6 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { isMetadataFile } from './_backupType.mjs'
import {
isKnownJournalEvent,
purgeBackupJournal,
readBackupJournal,
writeBackupJournalEntries,
writeBackupJournalEntry,
@@ -66,10 +65,6 @@ export class RemoteAdapter {
this._handler = handler
this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
this._useGetDiskLegacy = useGetDiskLegacy
// Cache of the partition list per diskId, populated by listPartitions and evicted by
// getDisk when the disk is unmounted
this._partitionsCache = new Map()
}
get handler() {
@@ -560,14 +555,6 @@ export class RemoteAdapter {
return readBackupJournal(this._handler, cursor, opts)
}
/**
* @param {number} before timestamp in ms, the journal days before its own are removed
* @returns {Promise<void>}
*/
async purgeBackupJournal(before) {
return purgeBackupJournal(this._handler, before)
}
// Same as `readBackupJournal()`, with the current metadata of the added and changed backups
// attached, so that a listing can be brought up to date from the result alone, in a single
// round-trip for a caller which is not on this host.

View File

@@ -393,47 +393,6 @@ describe('readBackupJournal()', { concurrency: 1 }, () => {
})
})
describe('purgeBackupJournal()', { concurrency: 1 }, () => {
const DAY = 24 * 60 * 60 * 1e3
const HOUR = 60 * 60 * 1e3
// midnight of the given number of days ago, so that the tests do not depend on the time they run at
const daysAgo = n => (Math.floor(Date.now() / DAY) - n) * DAY
const writeEntryAt = timestamp =>
writeJournalEntryAt(timestamp, { event: 'add', vmUuid, filename: `/${rootPath}/${timestamp}.json` })
test('removes the days before the one of `before`, and keeps that day whole', async () => {
for (const n of [3, 2]) {
await writeEntryAt(daysAgo(n) + HOUR)
}
// after the entry of its own day: that entry is kept all the same
await adapter.purgeBackupJournal(daysAgo(2) + 2 * HOUR)
assert.deepEqual(await listJournalDays(), [formatJournalDay(daysAgo(2))])
})
test('skips a day which cannot be removed, and removes the next ones', async t => {
for (const n of [3, 2]) {
await writeEntryAt(daysAgo(n))
}
const lockedDay = formatJournalDay(daysAgo(3))
const rmtree = handler.rmtree.bind(handler)
t.mock.method(handler, 'rmtree', async path => {
if (path === `/${BACKUP_JOURNAL_DIR}/${lockedDay}`) {
throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' })
}
return rmtree(path)
})
await adapter.purgeBackupJournal(Date.now())
assert.deepEqual(await listJournalDays(), [lockedDay])
})
})
describe('readBackupJournalEvents()', { concurrency: 1 }, () => {
// records the paths whose metadata is read from now on, to check a replay does not re-read the
// same backup once per event

View File

@@ -309,47 +309,3 @@ export async function readBackupJournal(handler, cursor, { mustExist = false } =
return entries
}
/**
* Removes the journal entries of the days before the one of `before`.
*
* Whole days are removed, entries are never removed one by one: the day of `before` is kept entirely,
* whatever the time of its entries.
*
* A day which cannot be removed is skipped, e.g. its files are immutable (`EPERM` on a local
* repository, `AccessDenied` from the policy of a bucket): it must not keep the later days from being
* removed, and the next purge tries it again. Only a failure to list the journal rejects.
*
* A purge warns once, with its first failure and their count: where deletes are refused, every day
* before `before` fails again on each purge, one more each day.
*
* @param {RemoteHandler} handler
* @param {number} before timestamp in ms
* @returns {Promise<void>}
*/
export async function purgeBackupJournal(handler, before) {
const minDay = formatJournalDay(before)
const days = await handler.list(`/${BACKUP_JOURNAL_DIR}`, {
filter: name => isDayName(name) && name < minDay,
ignoreMissing: true,
})
let firstFailure
let nFailures = 0
// one day at a time: this runs in the background, it must not compete with the backups for the
// remote
for (const day of days) {
const path = `/${BACKUP_JOURNAL_DIR}/${day}`
try {
await handler.rmtree(path)
debug('journal day purged', { path })
} catch (error) {
if (firstFailure === undefined) {
firstFailure = { error, path }
}
++nFailures
}
}
if (firstFailure !== undefined) {
warn('failed to purge some journal days', { ...firstFailure, nFailures })
}
}

View File

@@ -1,111 +0,0 @@
// @ts-check
import { createLogger } from '@xen-orchestra/log'
import moment from 'moment-timezone'
import { parse } from 'xo-remote-parser'
const { warn } = createLogger('xo:backups:buildContext')
/**
* @typedef {import('@vates/types/xen-api').XenApiVm} XenApiVm
* @typedef {import('@vates/types/xo').XoSr} XoSr
* @typedef {import('@vates/types/xo').XoBackupRepository} XoBackupRepository
*
* @typedef {object} RunContext
* @property {number} dayOfWeek
* @property {number} dayOfMonth
* @property {number} hour
* @property {number} month
* @property {number} year
*
* @typedef {object} VmContext
* @property {Record<string, unknown>} vm
* @property {RunContext} run
* @property {number} [chainLength]
*
* @typedef {object} SrContext
* @property {{name_label: string, type: string, description: string, tags: string[]}} sr
* @property {RunContext} run
*
* @typedef {object} BackupRepositoryContext
* @property {{name: string, type: string | undefined, tags: string[]}} backupRepository
* @property {RunContext} run
*/
/**
* @param {Date} date
* @param {string | undefined} timezone
* @returns {RunContext}
*/
export function buildRunContext(date, timezone) {
const momentDate = timezone ? moment.tz(date, timezone) : moment(date)
return {
dayOfWeek: momentDate.day(),
dayOfMonth: momentDate.date(),
hour: momentDate.hour(),
month: momentDate.month() + 1,
year: momentDate.year(),
}
}
/**
* @param {XenApiVm} vm
* @param {RunContext} run
* @param {number} [chainLength]
* @returns {VmContext}
*/
export function buildVmContext(vm, run, chainLength) {
/** @type {Record<string, unknown>} */
const vmObject = {}
for (const [key, value] of Object.entries(vm)) {
if (!key.startsWith('$') && typeof value !== 'function') {
vmObject[key] = value
}
}
if (!Array.isArray(vmObject.tags)) {
vmObject.tags = []
}
return { vm: vmObject, run, chainLength }
}
/**
* @param {XoSr} sr
* @param {RunContext} run
* @returns {SrContext}
*/
export function buildSrContext(sr, run) {
return {
sr: {
name_label: sr.name_label,
type: sr.SR_type,
description: sr.name_description,
tags: Array.isArray(sr.tags) ? sr.tags : [],
},
run,
}
}
/**
* @param {XoBackupRepository} backupRepository
* @param {RunContext} run
* @returns {BackupRepositoryContext}
*/
export function buildBackupRepositoryContext(backupRepository, run) {
let type
try {
type = parse(backupRepository.url).type
} catch (error) {
warn('cannot parse backup repository URL', { backupRepositoryId: backupRepository.id, error })
}
return {
backupRepository: {
name: backupRepository.name,
type,
tags: [], // Placeholder - Backup repositories don't have tags yet
},
run,
}
}

View File

@@ -1,147 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { buildRunContext, buildVmContext, buildSrContext, buildBackupRepositoryContext } from './_buildContext.mjs'
import { BACKUP_EXPRESSION_CONTEXT } from '@vates/types/backup-expression-context'
describe('buildContext', () => {
const date = new Date('2025-01-05T03:00:00Z')
const runContext = buildRunContext(date, 'America/New_York')
const expectedRunContext = {
dayOfWeek: 6,
dayOfMonth: 4,
hour: 22,
month: 1,
year: 2025,
}
it('buildRunContext', () => {
assert.deepEqual(runContext, expectedRunContext)
})
it('buildVmContext', () => {
const vm1 = { name: 'name', $value: '$value', func: () => {} }
const vmContext1 = buildVmContext(vm1, runContext, 3)
assert.deepEqual(vmContext1, {
vm: {
name: 'name',
tags: [],
},
run: expectedRunContext,
chainLength: 3,
})
const vm2 = { name: 'name', $value: '$value', func: () => {}, tags: ['tag'] }
const vmContext2 = buildVmContext(vm2, runContext, undefined)
assert.deepEqual(vmContext2, {
vm: {
name: 'name',
tags: ['tag'],
},
run: expectedRunContext,
chainLength: undefined,
})
const vm3 = { name: 'name', $value: '$value', func: () => {}, tags: 'tag' }
const vmContext3 = buildVmContext(vm3, runContext, undefined)
assert.deepEqual(vmContext3, {
vm: {
name: 'name',
tags: [],
},
run: expectedRunContext,
chainLength: undefined,
})
})
it('buildVmContext exposes every field advertised by the schema', () => {
// vm is an open context (buildVmContext copies through all real VM fields),
// so we assert the schema's curated fields are a subset of the output, not equality.
const vm = { name_label: 'name', power_state: 'Running', tags: ['tag'], $ref: 'ref', func: () => {} }
const { vm: vmContext } = buildVmContext(vm, runContext)
for (const field of Object.keys(BACKUP_EXPRESSION_CONTEXT.vm)) {
assert.ok(field in vmContext, `vm context is missing schema field: ${field}`)
}
})
it('buildSrContext', () => {
const sr1 = {
name_label: 'name_label',
SR_type: 'type',
name_description: 'description',
ignored: 'ignored',
}
const srContext1 = buildSrContext(sr1, runContext)
assert.deepEqual(srContext1, {
sr: {
name_label: 'name_label',
type: 'type',
description: 'description',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(Object.keys(srContext1.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
const sr2 = {
name_label: 'name_label',
SR_type: 'type',
name_description: 'description',
tags: ['tag'],
ignored: 'ignored',
}
const srContext2 = buildSrContext(sr2, runContext)
assert.deepEqual(srContext2, {
sr: {
name_label: 'name_label',
type: 'type',
description: 'description',
tags: ['tag'],
},
run: expectedRunContext,
})
assert.deepEqual(Object.keys(srContext2.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
})
it('buildBackupRepositoryContext', () => {
const backupRepository1 = {
name: 'name',
url: 'nfs://host/path',
ignored: 'ignored',
}
const backupRepositoryContext1 = buildBackupRepositoryContext(backupRepository1, runContext)
assert.deepEqual(backupRepositoryContext1, {
backupRepository: {
name: 'name',
type: 'nfs',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(
Object.keys(backupRepositoryContext1.backupRepository).sort(),
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
)
const backupRepository2 = {
name: 'name',
url: 'nfs://host/path',
tags: [],
ignored: 'ignored',
}
const backupRepositoryContext2 = buildBackupRepositoryContext(backupRepository2, runContext)
assert.deepEqual(backupRepositoryContext2, {
backupRepository: {
name: 'name',
type: 'nfs',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(
Object.keys(backupRepositoryContext2.backupRepository).sort(),
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
)
})
})

View File

@@ -1,24 +0,0 @@
// @ts-check
import * as CM from 'complex-matcher'
/**
* Parse and compile a complex-matcher expression into a predicate function.
* Throws synchronously if the expression is syntactically invalid.
*
* @param {string} expression
* @returns {(context: object) => boolean}
*/
export function compileExpression(expression) {
return CM.parse(expression).createPredicate()
}
/**
* Returns true if value is a non-empty string — i.e. a complex-matcher expression.
*
* @param {unknown} value
* @returns {value is string}
*/
export function isExpression(value) {
return typeof value === 'string' && value.length > 0 && !Number.isFinite(Number(value))
}

View File

@@ -1,95 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { compileExpression, isExpression } from './_expressionPredicate.mjs'
describe('compileExpression', () => {
it('run:dayOfWeek:0', () => {
const predicate = compileExpression('run:dayOfWeek:0')
assert(predicate({ run: { dayOfWeek: 0 } }))
assert(!predicate({ run: { dayOfWeek: 1 } }))
})
it('chainLength:>5', () => {
const predicate = compileExpression('chainLength:>5')
assert(predicate({ chainLength: 6 }))
assert(!predicate({ chainLength: 5 }))
})
it('vm:tags:prod', () => {
const predicate = compileExpression('vm:tags:prod')
assert(predicate({ vm: { tags: ['prod'] } }))
assert(predicate({ vm: { tags: ['prod', 'test'] } }))
assert(!predicate({ vm: { tags: ['test'] } }))
assert(!predicate({ vm: { tags: [] } }))
})
it('run:dayOfWeek:0 chainLength:>3', () => {
const predicate = compileExpression('run:dayOfWeek:0 chainLength:>3')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ run: { dayOfWeek: 0 }, chainLength: 2 }))
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
})
it('|(run:dayOfWeek:0 chainLength:>6)', () => {
const predicate = compileExpression('|(run:dayOfWeek:0 chainLength:>6)')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 7 }))
assert(predicate({ run: { dayOfWeek: 1 }, chainLength: 7 }))
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
})
it('missing field', () => {
const predicate = compileExpression('run:dayOfWeek:0')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ chainLength: 6 }))
assert(!predicate({ run: {}, chainLength: 6 }))
})
it('null-checking', () => {
const predicate = compileExpression('!vm')
assert(predicate({}))
assert(!predicate({ vm: [{ vm: 'vm' }] }))
})
it('throws on invalid expression', () => {
assert.throws(() => compileExpression('vm:('))
})
})
describe('isExpression', () => {
it('non-empty string', () => {
assert(isExpression('vm:tags:test'))
})
it('number', () => {
assert(!isExpression(0))
})
it('string[]', () => {
assert(!isExpression(['string']))
})
it('undefined', () => {
assert(!isExpression(undefined))
})
it('empty string', () => {
assert(!isExpression(''))
})
it('number', () => {
assert(!isExpression('3'))
})
it('number with spaces', () => {
assert(!isExpression(' 5 '))
})
})

View File

@@ -1,7 +1,7 @@
import { strict as assert } from 'node:assert'
import test from 'node:test'
import { execFile } from 'node:child_process'
import { copyFile, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { Stream } from 'node:stream'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -9,7 +9,6 @@ import { promisify } from 'node:util'
import { randomBytes } from 'node:crypto'
import Disposable from 'promise-toolbox/Disposable'
import * as tar from 'tar'
import { RemoteAdapter } from './RemoteAdapter.mjs'
@@ -72,83 +71,6 @@ test('fetchPartitionFiles resolves with a node:stream for every format', async t
}
})
async function readAll(stream) {
const chunks = []
for await (const chunk of stream) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
async function listTgzEntries(archive, tmp) {
const file = join(tmp, 'archive.tgz')
await writeFile(file, archive)
const entries = []
await tar.t({ file, onReadEntry: entry => entries.push(entry.path.replace(/\/$/, '')) })
return entries
}
// The partition is mounted on the XOA/proxy: a guest symlink is resolved against the host root,
// so following one would expose host files. Paths through a symlink are refused, and symlinks
// met while walking a directory are skipped.
test('file restore never follows symbolic links', async t => {
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-symlink-'))
t.after(() => rm(tmp, { recursive: true, force: true }))
// `host` stands for the XOA filesystem, `root` for the mounted guest partition
const host = join(tmp, 'host')
const root = join(tmp, 'root')
await mkdir(host)
await writeFile(join(host, 'secret.txt'), 'host secret')
await mkdir(join(root, 'dir'), { recursive: true })
await writeFile(join(root, 'dir', 'file.txt'), 'guest file')
await symlink(host, join(root, 'absolute')) // like `ln -s / foo` in the guest
await symlink('../../host', join(root, 'dir', 'relative'))
await symlink('file.txt', join(root, 'dir', 'in-tree'))
const adapter = new RemoteAdapter({})
adapter.getPartition = Disposable.factory(async function* () {
yield root
})
const refusedPaths = ['/absolute', '/absolute/secret.txt', '/dir/relative', '/dir/relative/host/secret.txt']
await t.test('listing refuses a path through a symlink', async () => {
for (const path of refusedPaths) {
await assert.rejects(adapter.listPartitionFiles('disk', 'partition', path), { code: 'ELINK_NOT_SUPPORTED' }, path)
}
})
await t.test('listing skips symlinks', async () => {
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/'), { 'dir/': {} })
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/dir'), { 'file.txt': {} })
})
for (const format of ['tgz', 'zip']) {
await t.test(`${format}: download refuses a path through a symlink`, async () => {
for (const path of refusedPaths) {
await assert.rejects(
adapter.fetchPartitionFiles('disk', 'partition', ['/dir/file.txt', path], format),
{ code: 'ELINK_NOT_SUPPORTED' },
path
)
}
})
}
await t.test('tgz: symlinks inside a selected directory are skipped', async () => {
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'tgz'))
assert.deepEqual((await listTgzEntries(archive, tmp)).sort(), ['dir', 'dir/file.txt'])
})
await t.test('zip: symlinks inside a selected directory are skipped', async () => {
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'zip'))
assert.ok(archive.includes('dir/file.txt'))
assert.equal(archive.includes('dir/relative'), false)
assert.equal(archive.includes('dir/in-tree'), false)
})
})
test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => {
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-'))
const adapter = new RemoteAdapter({})

View File

@@ -66,29 +66,7 @@ export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
}))
const makeRelative = path => resolve('/', path).slice(1)
// The partition is mounted on the XOA/proxy, so the kernel resolves a guest symlink against the
// host root: following one (e.g. `ln -s / foo` in the VM, then `/foo/etc`) would read host files.
// Symlinks are therefore not supported by file restore: a path going through one is refused,
// checked component by component with lstat. The mount is read-only, so no check/use race.
export async function resolveWithoutSymlinks(rootPath, path) {
const relativePath = makeRelative(path)
let currentPath = rootPath
for (const name of relativePath.split('/')) {
if (name === '') {
continue
}
currentPath = join(currentPath, name)
const stats = await lstat(currentPath)
if (stats.isSymbolicLink()) {
const error = new Error(`symbolic links are not supported by file restore: ${path}`)
error.code = 'ELINK_NOT_SUPPORTED'
error.path = path
throw error
}
}
return relativePath
}
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
for (const relativePath of relativePaths) {
@@ -312,12 +290,6 @@ export const fileRestoreMethods = {
const self = this
Disposable.use(async function* () {
const path = yield self.getPartition(diskId, partitionId)
// validated before resolving the promise, so a refused path rejects instead of
// producing a truncated archive
const relativePaths = []
for (const requestedPath of paths) {
relativePaths.push(await resolveWithoutSymlinks(path, requestedPath))
}
let outputStream
if (format === 'tgz') {
@@ -326,13 +298,7 @@ export const fileRestoreMethods = {
// simultaneous reads. Those saturate the libuv threadpool and starve
// the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
// threadpool deadlock). Serializing keeps a worker free for them.
//
// symlinks found while walking the selected directories are skipped, like in the zip
// format and the file listing
const pack = tar.c(
{ cwd: path, gzip: true, jobs: 1, filter: (_, stats) => !stats.isSymbolicLink() },
relativePaths
)
const pack = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
// node-tar >= 7 returns a Minipass stream: it has `pipe()` but is not a `node:stream`
// instance. xo-proxy serves this stream through Koa, which pipes only bodies passing
@@ -350,7 +316,7 @@ export const fileRestoreMethods = {
// appear as a freeze with no response sent.
outputStream = zip.outputStream
resolve(zip.outputStream)
await addZipEntries(zip, path, '', relativePaths)
await addZipEntries(zip, path, '', paths.map(makeRelative))
zip.end()
} else {
throw new Error('unsupported format ' + format)
@@ -365,23 +331,17 @@ export const fileRestoreMethods = {
},
async *getDisk(diskId) {
try {
if (this._useGetDiskLegacy) {
yield* getDiskLegacy(this._handler, diskId)
return
}
const handler = this._handler
// this is a disposable
const mountDir = yield getTmpDir()
// this is also a disposable
yield mount(handler, diskId, mountDir)
// this will yield disk path to caller
yield `${mountDir}/vhd0`
} finally {
// Disposable.factory disposes by calling gen.return(), which runs this finally before
// the mount/tmpdir disposers — i.e. exactly as the disk is unmounted.
this._partitionsCache.delete(diskId)
if (this._useGetDiskLegacy) {
yield* getDiskLegacy(this._handler, diskId)
return
}
const handler = this._handler
// this is a disposable
const mountDir = yield getTmpDir()
// this is also a disposable
yield mount(handler, diskId, mountDir)
// this will yield disk path to caller
yield `${mountDir}/vhd0`
},
// partitionId values:
@@ -411,7 +371,7 @@ export const fileRestoreMethods = {
listPartitionFiles(diskId, partitionId, path) {
return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
path = join(rootPath, await resolveWithoutSymlinks(rootPath, path))
path = resolveSubpath(rootPath, path)
const entriesMap = {}
await asyncEach(
await readdir(path),
@@ -437,13 +397,7 @@ export const fileRestoreMethods = {
},
listPartitions(diskId) {
const cache = this._partitionsCache
const cached = cache.get(diskId)
if (cached !== undefined) {
return cached
}
const pPartitions = Disposable.use(this.getDisk(diskId), async devicePath => {
return Disposable.use(this.getDisk(diskId), async devicePath => {
// partx may return empty on FUSE-backed files (vhd0); a loop device
// presents proper block-device semantics that partx reads reliably.
// losetup may itself fail if the FUSE mount isn't fully ready yet —
@@ -461,14 +415,11 @@ export const fileRestoreMethods = {
}
}
// direct listing failed, try with lvm listing
if (partitions.length === 0) {
debug('direct listing of the volume partition failed, try to read through LVM')
try {
// handle potential raw LVM physical volume
return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
} catch (error) {
warn('LVM listing of this volume failed ', error)
return []
}
}
@@ -504,16 +455,6 @@ export const fileRestoreMethods = {
})
return results
})
// The entry is evicted on disk unmount (see getDisk)
// a failed probe is not cached so the next call retries.
cache.set(diskId, pPartitions)
pPartitions.catch(() => {
if (cache.get(diskId) === pPartitions) {
cache.delete(diskId)
}
})
return pPartitions
},
}

View File

@@ -60,7 +60,7 @@ const parsePartxLine = createParser({
export async function listPartitions(devicePath) {
const parts = await fromCallback(execFile, 'partx', [
'--bytes',
'--output=START,SIZE,NAME,UUID,TYPE',
'--output=NR,START,SIZE,NAME,UUID,TYPE',
'--pairs',
devicePath,
]).catch(error => {

View File

@@ -10,8 +10,6 @@ import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
import { Throttle } from '@vates/generator-toolbox'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { compileExpression, isExpression } from '../_expressionPredicate.mjs'
import { buildRunContext, buildVmContext } from '../_buildContext.mjs'
const noop = Function.prototype
@@ -62,25 +60,21 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
return
}
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
const predicate = isExpression(settings.vmFilter) ? compileExpression(settings.vmFilter) : null
const vmsUuids = await sourceRemoteAdapter.listAllVms()
const queue = new Set(vmsUuids)
Task.info('vms', { vms: vmsUuids })
let nbVms = vmsUuids.length
const nbVms = vmsUuids.length
let nbVmsDone = 0
remoteAdapters = getAdaptersByRemote(remoteAdapters)
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmsUuids)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = async vmUuid => {
const handleVm = vmUuid => {
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
@@ -90,26 +84,12 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
const vmBackups = await sourceRemoteAdapter.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
// avoiding to create tasks for empty directories
if (vmBackups.length === 0) {
nbVms--
return
}
if (predicate !== null && !predicate(buildVmContext(vmBackups.at(-1).vm, runContext))) {
nbVms--
return
}
const opts = {
baseSettings,
config,
job,
healthCheckSr,
remoteAdapters,
runContext,
schedule,
settings: vmSettings,
sourceRemoteAdapter,
@@ -126,39 +106,45 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
}
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
let taskError
return task
.runInside(async () =>
vmBackup.run().catch(error => {
taskError = error
})
)
.then(result => {
if (taskError === undefined) {
nbVmsDone++
return task.success(result)
return sourceRemoteAdapter
.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
.then(vmBackups => {
// avoiding to create tasks for empty directories
if (vmBackups.length > 0) {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
let taskError
return task
.runInside(async () =>
vmBackup.run().catch(error => {
taskError = error
})
)
.then(result => {
if (taskError === undefined) {
nbVmsDone++
return task.success(result)
}
if (isLastRun) {
nbVmsDone++
return task.failure(taskError)
}
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
// don't end the task
task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: taskError.message,
isRetry: true,
})
queue.add(vmUuid)
})
.catch(noop)
}
if (isLastRun) {
nbVmsDone++
return task.failure(taskError)
}
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
// don't end the task
task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: taskError.message,
isRetry: true,
})
queue.add(vmUuid)
})
.catch(noop)
}
const { concurrency } = settings
const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)

View File

@@ -12,7 +12,6 @@ import { Throttle } from '@vates/generator-toolbox'
import { asyncEach } from '@vates/async-each'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
import { buildRunContext } from '../_buildContext.mjs'
const noop = Function.prototype
@@ -69,7 +68,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const getSnapshotNameLabel = this._getSnapshotNameLabel
const schedule = this._schedule
const settings = this._settings
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
const throttleGenerator = new Throttle(settings.maxExportRate)
const throttleStream = createStreamThrottle(settings.maxExportRate)
@@ -153,7 +151,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
healthCheckSr,
job,
remoteAdapters,
runContext,
schedule,
settings: vmSettings,
srs,
@@ -252,7 +249,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
throttleGenerator,
throttleStream,
vm,
runContext,
}
const vmBackup = this._getVmBackup(job.mode, opts)

View File

@@ -53,7 +53,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
'writer.run()'
)
// for healthcheck
this._vm = metadata.vm
this._tags = metadata.vm.tags
nbTransferredVms++
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
}

View File

@@ -131,7 +131,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
// for healthcheck
this._vm = metadata.vm
this._tags = metadata.vm.tags
nbTransferredVms++
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
}

View File

@@ -18,19 +18,10 @@ import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
import { buildVmContext } from '../../_buildContext.mjs'
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
constructor(props) {
super(props)
const fullInterval = this._settings.fullInterval
this._fullIntervalPredicate = isExpression(fullInterval) ? compileExpression(fullInterval) : null
}
_getWriters() {
return [
IncrementalRemoteWriter,
@@ -238,17 +229,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
}
// we do tafter checkbasevdis because we want the writer to know the target VM
// especially on replication when we always update the same VM
// especially on replication when we alwayr update the same VM
const fullInterval = this._settings.fullInterval
const chainLength = (deltaChainLength ?? 0) + 1
if (isExpression(fullInterval)) {
const context = buildVmContext(this._vm, this._runContext, chainLength)
if (this._fullIntervalPredicate(context)) {
debug('forcing full backup: fullInterval expression matched', { fullInterval })
return
}
} else if (fullInterval !== 0 && fullInterval <= chainLength) {
debug('not using base VM because fullInterval reached', { fullInterval, chainLength })
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
debug('not using base VM because fullInterval reached', {
fullInterval,
deltaChainLength,
eq: fullInterval < deltaChainLength + 1,
dc1: deltaChainLength + 1,
})
return
}

View File

@@ -1,8 +1,6 @@
import { asyncMap } from '@xen-orchestra/async-map'
import { createLogger } from '@xen-orchestra/log'
import { Task } from '@vates/task'
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
import { buildVmContext } from '../../_buildContext.mjs'
const { debug, warn } = createLogger('xo:backups:AbstractVmRunner')
@@ -75,58 +73,24 @@ export const Abstract = class AbstractVmBackupRunner {
}
// check if current VM has tags
const tags = this._vm.tags
const tags = this._tags
// accept both 'xo:no-health-check' and 'xo:no-health-check=reason'
if (tags.some(t => t.startsWith('xo:no-health-check'))) {
const vmAlwaysIgnored = tags.some(t => t.startsWith('xo:no-health-check'))
const intersect = !vmAlwaysIgnored && settings.healthCheckVmsWithTags.some(t => tags.includes(t))
if (vmAlwaysIgnored || (settings.healthCheckVmsWithTags.length !== 0 && !intersect)) {
// create a task to have an info in the logs and reports
return Task.run(
{
properties: {
name: 'health check',
},
properties: { name: 'health check' },
},
() => {
Task.info(`This VM is excluded from health checks via tag`)
Task.info(`This VM doesn't match the health check's tags for this schedule`)
}
)
}
if (isExpression(settings.healthCheckVmsWithTags)) {
const context = buildVmContext(this._vm, this._runContext)
if (!this._healthCheckPredicate(context)) {
return Task.run(
{
properties: { name: 'health check' },
},
() => {
Task.info(`VM did not match healthCheckVmsWithTags expression`)
}
)
}
} else {
const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
// create a task to have an info in the logs and reports
return Task.run(
{
properties: { name: 'health check' },
},
() => {
Task.info(`This VM doesn't match the health check's tags for this schedule`)
}
)
}
}
await this._callWriters(writer => writer.healthCheck(), 'writer.healthCheck()')
}
_compileHealthCheckPredicate() {
const { healthCheckVmsWithTags } = this._settings
if (isExpression(healthCheckVmsWithTags)) {
this._healthCheckPredicate = compileExpression(healthCheckVmsWithTags)
} else {
this._healthCheckPredicate = undefined
}
}
}

View File

@@ -20,7 +20,6 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
job,
healthCheckSr,
remoteAdapters,
runContext,
schedule,
settings,
sourceRemoteAdapter,
@@ -41,14 +40,11 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this._throttleGenerator = throttleGenerator
this._throttleStream = throttleStream
this._vmUuid = vmUuid
this._runContext = runContext
const allSettings = job.settings
const writers = new Set()
this._writers = writers
this._compileHealthCheckPredicate()
const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
if (settings.distributeBackups) {
writers.add(

View File

@@ -36,7 +36,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
job,
remoteAdapters,
remotes,
runContext,
schedule,
settings,
srs,
@@ -64,13 +63,12 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
this.timestamp = undefined
// VM currently backed up
const tags = vm.tags
const tags = (this._tags = vm.tags)
// VM (snapshot) that is really exported
this._exportedVm = undefined
this._synchronizedSnapshotTimestamp = synchronizedSnapshotTimestamp
this._vm = vm
this._runContext = runContext
this._getSnapshotNameLabel = getSnapshotNameLabel
this._isIncremental = job.mode === 'delta'
@@ -92,9 +90,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
settings.offlineSnapshot = true
}
this._settings = settings
this._compileHealthCheckPredicate()
// Create writers
{
const writers = new Set()

View File

@@ -65,11 +65,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
const snapshotCandidates = new Map()
// skip orphan snapshots (active VDI destroyed): they can't be validated against an active disk
const replicaSnapshots = sr.$VDIs.filter(
vdi => vdi?.managed && vdi?.is_a_snapshot && vdi.$snapshot_of !== undefined
)
for (const [baseUuid, srcVdiuid] of baseUuidToSrcVdi) {
let target
@@ -77,9 +72,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
if (contentKey !== undefined) {
debug('got one content key, look for a candidate')
target = replicaSnapshots.find(
target = sr.$VDIs.find(
vdi =>
vdi.other_config[CONTENT_KEY] === contentKey &&
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[CONTENT_KEY] === contentKey && // &&
// ensure we don't replicate on ourself or any of the vdi in the source chain
vdi.$snapshot_of.uuid !== srcVdiuid
)
@@ -88,11 +85,13 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
// fall back for older snapshots
if (target === undefined) {
debug('content key not here or not found , look by jobid ')
target = replicaSnapshots.find(
target = sr.$VDIs.find(
vdi =>
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[JOB_ID] === this._job.id &&
vdi.other_config[VM_UUID] === this._vmUuid &&
vdi.other_config[COPY_OF] === baseUuid
vdi?.other_config[COPY_OF] === baseUuid
)
}
if (target !== undefined) {

View File

@@ -1,26 +0,0 @@
import { isExpression, compileExpression } from './_expressionPredicate.mjs'
import { invalidParameters } from 'xo-common/api-errors.js'
export function assertValidSettingsExpression(settings, keys) {
for (const [scheduleId, scheduleSettings] of Object.entries(settings ?? {})) {
for (const key of keys) {
assertValidExpression(`settings/${scheduleId === '' ? 'general' : scheduleId}/${key}`, scheduleSettings?.[key])
}
}
}
export function assertValidExpression(name, value) {
if (isExpression(value)) {
try {
compileExpression(value)
} catch (error) {
throw invalidParameters(`Invalid ${name} expression: ${error.message}`, [
{ instancePath: `/${name}`, message: error.message },
])
}
} else if (typeof value === 'string' && value.trim() === '') {
throw invalidParameters(`Invalid ${name} expression: must be a non-empty expression`, [
{ instancePath: `/${name}`, message: 'must be a non-empty expression' },
])
}
}

View File

@@ -1,77 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { invalidParameters } from 'xo-common/api-errors.js'
import { assertValidExpression, assertValidSettingsExpression } from './backupExpressions.mjs'
const VALID = 'vm:tags:prod'
const INVALID = 'vm:('
// checks that the error is `invalidParameters` and points to the expected field
const isInvalidAt = instancePath => error =>
invalidParameters.is(error) && error.data.errors[0].instancePath === instancePath
describe('assertValidExpression', function () {
it('accepts a valid expression', function () {
assert.doesNotThrow(() => assertValidExpression('srs', VALID))
})
it('rejects an invalid expression with invalidParameters', function () {
assert.throws(() => assertValidExpression('srs', INVALID), isInvalidAt('/srs'))
})
it('ignores values which are not expressions', function () {
for (const value of [undefined, '3', { id: 'sr-1' }, { id: { __or: ['sr-1', 'sr-2'] } }]) {
assert.doesNotThrow(() => assertValidExpression('srs', value))
}
})
it('rejects blank strings', function () {
for (const value of ['', ' ']) {
assert.throws(() => assertValidExpression('srs', value), isInvalidAt('/srs'))
}
})
})
describe('assertValidSettingsExpression', function () {
const KEYS = ['healthCheckVmsWithTags', 'fullInterval']
it('accepts valid expressions in the general and the schedule settings', function () {
assert.doesNotThrow(() =>
assertValidSettingsExpression(
{ '': { fullInterval: VALID }, 'schedule-1': { healthCheckVmsWithTags: VALID } },
KEYS
)
)
})
it('rejects an invalid expression in the schedule settings and points to it', function () {
assert.throws(
() => assertValidSettingsExpression({ '': {}, 'schedule-1': { fullInterval: INVALID } }, KEYS),
isInvalidAt('/settings/schedule-1/fullInterval')
)
})
it('rejects an invalid expression in the general settings', function () {
// the general settings are keyed by '', named "general" in the path.
assert.throws(
() => assertValidSettingsExpression({ '': { healthCheckVmsWithTags: INVALID } }, KEYS),
isInvalidAt('/settings/general/healthCheckVmsWithTags')
)
})
it('ignores the legacy array and number values', function () {
assert.doesNotThrow(() =>
assertValidSettingsExpression({ 'schedule-1': { healthCheckVmsWithTags: ['tag'], fullInterval: 3 } }, KEYS)
)
})
it('ignores the keys it has not been asked to check', function () {
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': { vmFilter: INVALID } }, KEYS))
})
it('handles missing settings and undefined entries', function () {
assert.doesNotThrow(() => assertValidSettingsExpression(undefined, KEYS))
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': undefined }, KEYS))
})
})

View File

@@ -43,7 +43,7 @@ A VM directory holds **both** the full and the incremental backups of that VM: t
Transient entries, not listed in the trees below:
- `xo-vm-backups/<VM UUID>.lock`, held while a job writes to that VM's folder and while `cleanVm` cleans or merges it (after a job, after a deletion, or in the merge worker on local remotes). It is effective on S3 (an object rewritten every 30 seconds) and Azure (blob lease) too. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
- `xo-vm-backups/<VM UUID>.lock`, held for the duration of a job run on that VM. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
- `xo-vm-backups/.queue/clean-vm/<YYYYMMDD>T<HHmmss>-<random>`, the merge worker queue
- `.<VHD file name>.merge.json`, in a VDI directory, interrupted merge states.
@@ -167,7 +167,6 @@ Append-only log of what happened to the VM backup metadata.
- entries are never modified nor overwritten, so it works on an immutable remote
- written unencrypted, so it stays readable without the encryption key
- one directory per day: a reader lists only the days it misses, a purge drops whole directories
- xo-server removes the days older than `backups.journalRetention` from every enabled remote when it starts, then once a day; it always keeps the `fullListingInterval` of the remote, during which it reads the journal back; a day which cannot be removed is skipped, and the next purge tries it again
- the time part has a fixed width, so sorting the names by name sorts them by date
- `event` is `add`, `change` or `del`, and the file content also has a `reason` field
- the name of the file is only there to be read by a human: XO reads every field from the file content

View File

@@ -40,7 +40,6 @@
"@xen-orchestra/backup-archive": "^1.2.0",
"app-conf": "^3.0.0",
"compare-versions": "^6.0.0",
"complex-matcher": "^1.1.1",
"d3-time-format": "^4.1.0",
"decorator-synchronized": "^0.6.0",
"golike-defer": "^0.5.1",
@@ -58,8 +57,6 @@
"value-matcher": "^0.2.0",
"vhd-lib": "^4.17.0",
"xen-api": "^5.1.0",
"xo-common": "^0.11.0",
"xo-remote-parser": "^2.0.0",
"yazl": "^2.5.1"
},
"devDependencies": {

View File

@@ -9,7 +9,7 @@ async function consumeGenerator(source: AsyncGenerator) {
return 'finished'
}
test('Disk class', async t => {
const testRes = await t.test('synchronized fork disk ', async () => {
const testRes = await t.test('synchronised fork disk ', async () => {
const blockSize = 1024
const nbBlocks = 20
const source = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })

View File

@@ -18,7 +18,7 @@ ee.on('start', async function () {
// listeners have settled
await ee.emitAsync('start')
// by default, it will rejects as soon as one listener reject, you can customize
// by default, it will rejects as soon as one listener reject, you can customise
// error handling though:
await ee.emitAsync(
{

View File

@@ -36,7 +36,7 @@ ee.on('start', async function () {
// listeners have settled
await ee.emitAsync('start')
// by default, it will rejects as soon as one listener reject, you can customize
// by default, it will rejects as soon as one listener reject, you can customise
// error handling though:
await ee.emitAsync(
{

View File

@@ -1,4 +0,0 @@
// the error of proper-lockfile, which LocalHandler uses: callers and the documentation know this one
export default function lockedError(path) {
return Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED', file: path })
}

View File

@@ -1,12 +1,9 @@
import { BlobServiceClient, StorageSharedKeyCredential } from '@azure/storage-blob'
import { createLogger } from '@xen-orchestra/log'
import { pTimeout } from 'promise-toolbox'
import { parse } from 'xo-remote-parser'
import { join, split } from './path'
import RemoteHandlerAbstract from './abstract'
import copyStreamToBuffer from './_copyStreamToBuffer'
import lockedError from './_lockedError.js'
import { randomUUID } from 'node:crypto'
import { PassThrough, Transform, pipeline } from 'stream'
createLogger('xo:fs:azure')
@@ -14,52 +11,7 @@ const MAX_BLOCK_SIZE = 1024 * 1024 * 4000 // 4000 MiB
const MIN_BLOCK_SIZE = 1024 * 1024 * 4 // 4 MiB, default max block size
const MAX_BLOCK_COUNT = 50000
const AZURE_BATCH_MAX_REQUEST = 256 // BATCH_MAX_REQUEST constant from Azure
const { debug, warn, info } = createLogger('xo:fs:azure')
// how long the lease of a lock lasts without being renewed, in seconds: the longest finite lease Azure allows, as an
// infinite one would outlive a holder which crashed, and block the path until someone breaks it by hand
export const LEASE_DURATION = 60
// how often a holder renews the lease of its lock: 4 renewals per lease, so that a holder survives a few slow or
// failed requests without losing its lock
export const RENEW_INTERVAL = 15e3
// how long a renewal, or a reacquisition, waits for its requests: a hung one counts as a failed one, and the next
// renewal must still run before the lease expires, so RENEW_INTERVAL + RENEW_TIMEOUT + RENEW_INTERVAL < LEASE_DURATION
// (15 s + 20 s + 15 s < 60 s)
export const RENEW_TIMEOUT = 20e3
// leases the blob, creating it first, or throws `lockedError(path)`
//
// The holder of the lock deletes the blob when it releases it: when that happens between the upload and the
// acquisition, one more attempt. Losing a second race means another contender got it.
async function takeLease(blobClient, leaseClient, path) {
for (let attempt = 0; attempt < 2; ++attempt) {
try {
await blobClient.upload('', 0, { conditions: { ifNoneMatch: '*' } })
} catch (error) {
// 409: the blob exists; 412: it exists and is leased, and a write without the lease ID is refused
if (error.statusCode !== 409 && error.statusCode !== 412) {
throw error
}
}
try {
await leaseClient.acquireLease(LEASE_DURATION)
return
} catch (error) {
// 409: another lease is active
if (error.statusCode === 409) {
throw lockedError(path)
}
// 404: the blob was deleted since the upload
if (error.statusCode !== 404) {
throw error
}
}
}
throw lockedError(path)
}
const { warn, info } = createLogger('xo:fs:azure')
export default class AzureHandler extends RemoteHandlerAbstract {
#container
@@ -378,89 +330,4 @@ export default class AzureHandler extends RemoteHandlerAbstract {
await blobBatchClient.submitBatch(batch)
}
}
// Locks with a blob lease, Azure's own lock on a blob: Azure enforces its expiry, so there is no clock to compare,
// and a renewal writes nothing, where rewriting the blob would leave a copy every time on accounts with versioning
// or soft delete. An acquisition does leave one copy per lock taken, as the blob is created then deleted on release:
// a lease needs a blob to hold it.
//
// The requests are sent with the blob client directly, not through the handler's methods: those go through the
// handler's concurrency limit, and a renewal queued behind long uploads would let the lease expire.
//
// As with `LocalHandler#_lock()`, losing the lock while holding it is only logged, and the release function never
// rejects. A renewal which finds the lease gone takes the lock back, and the lock is given up only when another
// holder has it: any other error is a failed request, and the next renewal tries again.
async _lock(path) {
// `<path>.lock`, the name of the local lock directory: the listings skip it already
const blobClient = this.#containerClient.getBlockBlobClient(this.#makeFullPath(`${path}.lock`))
// Azure requires a GUID. Proposing it, rather than letting Azure pick one, makes the retry of the SDK harmless
// when the response of an acquisition is lost: acquiring with the ID of the active lease succeeds
const leaseId = randomUUID()
const leaseClient = blobClient.getBlobLeaseClient(leaseId)
await takeLease(blobClient, leaseClient, path)
let lastRenewal = Date.now()
let lost = false
let released = false
let renewing = Promise.resolve()
let timeout
async function renew() {
try {
await pTimeout.call(leaseClient.renewLease(), RENEW_TIMEOUT)
lastRenewal = Date.now()
return
} catch (error) {
// 404: the blob is gone; 409: it is leased to someone else
if (error.statusCode !== 404 && error.statusCode !== 409) {
// a failed request is not a lost lock: it is lost only if the lease expires and someone takes it, which the
// next successful request finds out; it repeats every interval while Azure fails, hence debug
debug('lock could not be renewed', { error, path, sinceLastRenewal: Date.now() - lastRenewal })
return
}
}
warn('lock compromised', { path })
try {
await pTimeout.call(takeLease(blobClient, leaseClient, path), RENEW_TIMEOUT)
lastRenewal = Date.now()
info('compromised lock was reacquired', { path })
} catch (error) {
lost = error.code === 'ELOCKED'
warn('compromised lock could not be reacquired', { error, path })
}
}
function schedule() {
timeout = setTimeout(() => {
renewing = renew().then(() => {
if (!released && !lost) {
schedule()
}
})
}, RENEW_INTERVAL)
timeout.unref()
}
schedule()
return async () => {
released = true
clearTimeout(timeout)
try {
// a renewal in flight which found the blob deleted would take the lock again
await renewing
// only deletes the blob while the lease is ours: after a lost lock, it is someone else's
await blobClient.delete({ conditions: { leaseId } })
} catch (error) {
// 404: the blob is gone; 412: it is not leased to `leaseId`
if (error.statusCode !== 404 && error.statusCode !== 412) {
warn('lock could not be released', { error, path })
}
}
}
}
}

View File

@@ -1,252 +0,0 @@
import { after, afterEach, before, beforeEach, describe, it, mock } from 'node:test'
import { strict as assert } from 'assert'
import 'dotenv/config'
import sinon from 'sinon'
import { ContainerClient } from '@azure/storage-blob'
import { getHandler } from '.'
import AzureHandler, { LEASE_DURATION, RENEW_INTERVAL, RENEW_TIMEOUT } from './azure.js'
// needs a real container, on Azure or Azurite
//
// xo_fs_azure='azure://<account>:<key>@<host>/<container>/<dir>' yarn workspace @xen-orchestra/fs test
const url = process.env.xo_fs_azure
describe('AzureHandler#lock()', { skip: url === undefined }, () => {
let handler
before(async () => {
handler = getHandler({ url }).addPrefix(`xo-fs-tests-${Date.now()}`)
await handler.sync()
})
after(async () => {
await handler?.forget()
})
it('refuses a held lock until it is released', async () => {
const lock = await handler.lock('vm')
try {
await assert.rejects(handler.lock('vm'), { code: 'ELOCKED' })
} finally {
await lock.dispose()
}
const again = await handler.lock('vm')
await again.dispose()
})
})
// `AzureHandler#_lock()` against a stubbed blob client: each status code it maps is checked through the lock it takes
describe('AzureHandler#_lock with stubbed blob client', () => {
let blob
let leaseClient
let getBlockBlobClientStub
let handler
const httpError = statusCode => Object.assign(new Error(`HTTP ${statusCode}`), { statusCode })
// one renewal, after `RENEW_INTERVAL`
async function renewOnce() {
mock.timers.tick(RENEW_INTERVAL)
await new Promise(resolve => setImmediate(resolve))
}
// a request which hangs until `RENEW_TIMEOUT` gives up on it
async function timeOut() {
mock.timers.tick(RENEW_TIMEOUT)
await new Promise(resolve => setImmediate(resolve))
}
beforeEach(() => {
mock.timers.enable({ apis: ['setTimeout', 'Date'] })
leaseClient = {
acquireLease: sinon.stub().resolves(),
renewLease: sinon.stub().resolves(),
}
blob = {
upload: sinon.stub().resolves(),
delete: sinon.stub().resolves(),
getBlobLeaseClient: sinon.stub().returns(leaseClient),
}
getBlockBlobClientStub = sinon.stub(ContainerClient.prototype, 'getBlockBlobClient').returns(blob)
handler = new AzureHandler({ url: 'azure://account:a2V5@account.blob.core.windows.net/container/dir' })
})
afterEach(() => {
getBlockBlobClientStub.restore()
mock.timers.reset()
})
it('leases the blob next to the path, under the directory of the handler, with a GUID of its own', async () => {
const release = await handler._lock('vm')
assert.deepEqual(getBlockBlobClientStub.firstCall.args, ['dir/vm.lock'])
assert.match(
blob.getBlobLeaseClient.firstCall.args[0],
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
)
assert.deepEqual(leaseClient.acquireLease.firstCall.args, [LEASE_DURATION])
await release()
})
describe('create', () => {
// 409: the blob exists; 412: it exists and is leased
for (const statusCode of [409, 412]) {
it(`goes on to acquire the lease when create fails with ${statusCode}`, async () => {
blob.upload.rejects(httpError(statusCode))
const release = await handler._lock('vm')
assert.deepEqual(blob.upload.firstCall.args, ['', 0, { conditions: { ifNoneMatch: '*' } }])
assert.equal(leaseClient.acquireLease.callCount, 1)
await release()
})
}
it('rethrows an unexpected create error', async () => {
blob.upload.rejects(httpError(500))
await assert.rejects(handler._lock('vm'), { statusCode: 500 })
assert.equal(leaseClient.acquireLease.callCount, 0)
})
})
describe('acquire', () => {
it('reports a lease held by another as ELOCKED', async () => {
leaseClient.acquireLease.rejects(httpError(409))
await assert.rejects(handler._lock('vm'), { code: 'ELOCKED' })
})
it('retries once when the blob is gone, then gives up as ELOCKED', async () => {
leaseClient.acquireLease.rejects(httpError(404))
await assert.rejects(handler._lock('vm'), { code: 'ELOCKED' })
assert.equal(blob.upload.callCount, 2)
assert.equal(leaseClient.acquireLease.callCount, 2)
})
it('takes the lock when the blob is back on the retry', async () => {
leaseClient.acquireLease.onFirstCall().rejects(httpError(404))
const release = await handler._lock('vm')
assert.equal(blob.upload.callCount, 2)
await release()
})
it('rethrows an unexpected acquire error', async () => {
leaseClient.acquireLease.rejects(httpError(500))
await assert.rejects(handler._lock('vm'), { statusCode: 500 })
})
})
describe('renew', () => {
// 404: the blob is gone; 409: it is leased to someone else. Either way the lock is taken again
for (const statusCode of [404, 409]) {
it(`takes the lock again when renewal fails with ${statusCode}`, async () => {
const release = await handler._lock('vm')
leaseClient.renewLease.rejects(httpError(statusCode))
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 1)
assert.equal(leaseClient.acquireLease.callCount, 2)
await release()
})
}
it('keeps the lock and renews again when renewal fails with another error', async () => {
const release = await handler._lock('vm')
leaseClient.renewLease.rejects(httpError(500))
await renewOnce()
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 2)
assert.equal(leaseClient.acquireLease.callCount, 1)
await release()
})
it('gives up on a renewal which hangs, and renews again', async () => {
const release = await handler._lock('vm')
leaseClient.renewLease.onFirstCall().returns(new Promise(() => {}))
await renewOnce()
await timeOut()
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 2)
assert.equal(leaseClient.acquireLease.callCount, 1)
await release()
})
it('gives up on a reacquisition which hangs, and renews again', async () => {
const release = await handler._lock('vm')
leaseClient.renewLease.onFirstCall().rejects(httpError(409))
leaseClient.acquireLease.onSecondCall().returns(new Promise(() => {}))
await renewOnce()
await timeOut()
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 2)
assert.equal(leaseClient.acquireLease.callCount, 2)
await release()
})
it('stops renewing once released', async () => {
const release = await handler._lock('vm')
await renewOnce()
await release()
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 1)
})
it('gives up the lock when another holder has the lease', async () => {
const release = await handler._lock('vm')
leaseClient.renewLease.rejects(httpError(409))
leaseClient.acquireLease.rejects(httpError(409))
await renewOnce()
await renewOnce()
assert.equal(leaseClient.renewLease.callCount, 1)
assert.equal(leaseClient.acquireLease.callCount, 2)
await release()
})
it('waits for a renewal in flight before releasing', async () => {
const release = await handler._lock('vm')
let renewed
leaseClient.renewLease.returns(new Promise(resolve => (renewed = resolve)))
await renewOnce()
const releasing = release()
await new Promise(resolve => setImmediate(resolve))
assert.equal(blob.delete.callCount, 0)
renewed()
await releasing
assert.equal(blob.delete.callCount, 1)
})
})
describe('remove', () => {
it('deletes the blob only while the lease is ours', async () => {
const release = await handler._lock('vm')
await release()
const [leaseId] = blob.getBlobLeaseClient.firstCall.args
assert.deepEqual(blob.delete.firstCall.args, [{ conditions: { leaseId } }])
})
// 404: the blob is gone; 412: it is not leased to us. Both are fine, and release never rejects anyway
for (const statusCode of [404, 412, 500]) {
it(`releases without rejecting when delete fails with ${statusCode}`, async () => {
const release = await handler._lock('vm')
blob.delete.rejects(httpError(statusCode))
await release()
assert.equal(blob.delete.callCount, 1)
})
}
})
})

View File

@@ -19,7 +19,6 @@ import { getApplyMd5BodyChecksumPlugin } from '@aws-sdk/middleware-apply-body-ch
import { Agent as HttpAgent } from 'http'
import { Agent as HttpsAgent } from 'https'
import { createLogger } from '@xen-orchestra/log'
import lockedError from './_lockedError.js'
import { PassThrough, Transform, pipeline } from 'stream'
import { parse } from 'xo-remote-parser'
import copyStreamToBuffer from './_copyStreamToBuffer.js'
@@ -37,13 +36,6 @@ const MAX_PART_NUMBER = 10000
const MIN_PART_SIZE = 5 * 1024 * 1024
const { debug, info, warn } = createLogger('xo:fs:s3')
// how often a holder rewrites its lock object
export const REFRESH_INTERVAL = 30e3
// how long a lock object may go without being rewritten before a contender takes it over: a holder survives a few
// failed refreshes, and one which crashed blocks the path for 3 minutes at most
export const STALE_AFTER = 180e3
export default class S3Handler extends RemoteHandlerAbstract {
#bucket
#dir
@@ -51,8 +43,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
#httpAgent
#httpsAgent
#immutable = false
// the S3 user may not read the bucket's Object Lock configuration: whether it is enabled is unknown
#objectLockUnknown = false
#maxPartSize
#maxPartNumber
#minPartSize
@@ -524,12 +514,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
if (error.$metadata.httpStatusCode === 403) {
info(`s3 user doesnt have enough privilege to check for Object Lock, enable content MD5 header`)
this.#s3.middlewareStack.use(getApplyMd5BodyChecksumPlugin(this.#s3.config))
if (!this.#objectLockUnknown) {
warn(
'the S3 user cannot read the Object Lock configuration of the bucket (s3:GetBucketObjectLockConfiguration), nothing is locked on this remote'
)
}
this.#objectLockUnknown = true
} else if (error.Code === 'ObjectLockConfigurationNotFoundError' || error.$metadata.httpStatusCode === 501) {
info(`Object lock is not available or not configured, don't add the content MD5 header`)
} else {
@@ -537,77 +521,4 @@ export default class S3Handler extends RemoteHandlerAbstract {
}
}
}
// Object Lock buckets keep the no-op lock of `RemoteHandlerAbstract`: every write of the lock object would leave
// a version retained for the whole retention period. So do the buckets whose Object Lock configuration the S3
// user cannot read (see `_sync()`).
//
// The lock is an object rewritten with plain writes, which every S3 provider supports: it is held while it is
// fresher than STALE_AFTER. It is not atomic: two contenders which find it free at the same time both take it, and
// a holder whose refreshes failed for STALE_AFTER keeps writing in a folder someone else took over.
//
// Its age is read on the provider's clock alone, `Last-Modified` against the `Date` of the response: XO's clock may
// not agree with the provider's, and one more than 2.5 minutes ahead would take over a lock its holder keeps
// refreshing (the SDK corrects its own clock only from 5 minutes off).
async _lock(path) {
if (this.isImmutable() || this.#objectLockUnknown) {
return super._lock(path)
}
// `<path>.lock`, the name of the local lock directory: the listings skip it already
const params = this.#createParams(`${path}.lock`)
// the requests are sent with the S3 client directly, not through the handler's methods: those go through the
// handler's concurrency limit, and a refresh queued behind long uploads would let the lock go stale
const write = () => this.#s3.send(new PutObjectCommand({ ...params, Body: '' }))
// the output of a HEAD has no `Date`: it is read from the response
let date
const head = new HeadObjectCommand(params)
head.middlewareStack.add(
next => async args => {
const result = await next(args)
date = result.response.headers.date
return result
},
{ step: 'deserialize' }
)
let current
try {
current = await this.#s3.send(head)
} catch (error) {
if (error.name !== 'NotFound') {
throw error
}
}
// a provider which omits `Last-Modified` or `Date` never lets a lock be taken over: it stays held until released
const age = Date.parse(date) - current?.LastModified?.getTime()
if (current !== undefined && (Number.isNaN(age) || age <= STALE_AFTER)) {
throw lockedError(path)
}
await write()
let refreshing = Promise.resolve()
const interval = setInterval(() => {
refreshing = write().then(
() => {},
// a failed refresh is not a lost lock: the next one tries again
error => debug('lock could not be refreshed', { error, path })
)
}, REFRESH_INTERVAL)
interval.unref()
return async () => {
clearInterval(interval)
try {
// a refresh landing after the delete would write the lock again
await refreshing
await this.#s3.send(new DeleteObjectCommand(params))
} catch (error) {
warn('lock could not be released', { error, path })
}
}
}
}

View File

@@ -1,216 +0,0 @@
import { after, before, describe, it, beforeEach, afterEach, mock } from 'node:test'
import { strict as assert } from 'assert'
import 'dotenv/config'
import sinon from 'sinon'
import {
DeleteObjectCommand,
GetObjectLockConfigurationCommand,
HeadObjectCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3'
import { getHandler } from '.'
import S3Handler, { REFRESH_INTERVAL, STALE_AFTER } from './s3.js'
// needs a real bucket without Object Lock: on an Object Lock bucket the lock is the no-op one, the second
// `lock()` succeeds, and this test fails
//
// xo_fs_s3='s3://<key>:<secret>@<host>/<bucket>/<dir>' yarn workspace @xen-orchestra/fs test
const url = process.env.xo_fs_s3
describe('S3Handler#lock()', { skip: url === undefined }, () => {
let handler
before(async () => {
handler = getHandler({ url }).addPrefix(`xo-fs-tests-${Date.now()}`)
await handler.sync()
})
after(async () => {
await handler?.forget()
})
it('refuses a held lock until it is released', async () => {
const lock = await handler.lock('vm')
try {
await assert.rejects(handler.lock('vm'), { code: 'ELOCKED' })
} finally {
await lock.dispose()
}
const again = await handler.lock('vm')
await again.dispose()
})
})
const createHandler = () => new S3Handler({ url: 's3://key:secret@localhost/bucket/dir' })
const s3Error = (name, httpStatusCode) => Object.assign(new Error(name), { name, $metadata: { httpStatusCode } })
// lets the async callback of a timer which just fired run to its end
const flush = () => new Promise(resolve => setImmediate(resolve))
const KEY = 'dir/vm.lock'
// `S3Handler#_lock()` against a stubbed `S3Client#send()`
describe('S3Handler#_lock() with a stubbed S3Client', () => {
// by key
let objects
// what the provider does wrong, which each test sets
let faults
// how far the provider's clock is ahead of XO's
let clockOffset
let send
const providerDate = () => new Date(Date.now() + clockOffset)
beforeEach(() => {
mock.timers.enable({ apis: ['setInterval', 'Date'] })
objects = new Map()
faults = {}
clockOffset = 0
send = sinon.stub(S3Client.prototype, 'send').callsFake(async command => {
const { Key } = command.input
const current = objects.get(Key)
if (command instanceof PutObjectCommand) {
if (faults.putStatus !== undefined) {
throw s3Error('InternalError', faults.putStatus)
}
// resolved by the test
if (faults.pendingPut !== undefined) {
await faults.pendingPut
}
objects.set(Key, { lastModified: providerDate() })
return {}
}
if (command instanceof HeadObjectCommand) {
if (current === undefined) {
throw s3Error('NotFound', 404)
}
// the lock reads the `Date` of the response in a middleware of the command, which this stub bypasses: run it
// on a response of its own
await command.middlewareStack.resolve(
async () => ({ response: { headers: { date: providerDate().toUTCString() } } }),
{}
)({ input: command.input })
return { LastModified: current.lastModified }
}
if (command instanceof DeleteObjectCommand) {
objects.delete(Key)
return {}
}
if (command instanceof GetObjectLockConfigurationCommand) {
if (faults.objectLock) {
return { ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }
}
throw s3Error('AccessDenied', 403)
}
throw new Error(`unexpected ${command.constructor.name}`)
})
})
afterEach(() => {
send.restore()
mock.timers.reset()
})
const writes = () => send.getCalls().filter(call => call.args[0] instanceof PutObjectCommand).length
for (const [what, objectLock] of [
['on an Object Lock bucket', true],
['when the S3 user cannot read the Object Lock configuration', false],
]) {
it(`takes no lock ${what}`, async () => {
faults.objectLock = objectLock
const handler = createHandler()
await handler._sync()
assert.equal(handler.isImmutable(), objectLock)
const release = await handler._lock('vm')
await release()
// the only request read the Object Lock configuration
assert.equal(send.callCount, 1)
})
}
it('takes <path>.lock in the directory of the remote, rewrites it every REFRESH_INTERVAL, and deletes it', async () => {
const release = await createHandler()._lock('vm')
assert.equal(objects.get(KEY).lastModified.getTime(), Date.now())
mock.timers.tick(REFRESH_INTERVAL)
await flush()
assert.equal(writes(), 2)
assert.equal(objects.get(KEY).lastModified.getTime(), Date.now())
await release()
assert.equal(objects.has(KEY), false)
})
it('refuses a held lock, and leaves it untouched', async () => {
const handler = createHandler()
const release = await handler._lock('vm')
try {
await assert.rejects(handler._lock('vm'), { code: 'ELOCKED' })
assert.equal(writes(), 1)
} finally {
await release()
}
})
it("takes over a stale lock, not a fresh one, on the provider's clock", async () => {
// XO's clock 2 minutes ahead: on it, the lock would look stale after 1 minute
clockOffset = -120e3
objects.set(KEY, { lastModified: providerDate() })
mock.timers.tick(STALE_AFTER)
await assert.rejects(createHandler()._lock('vm'), { code: 'ELOCKED' })
// the resolution of `Date`
mock.timers.tick(1e3)
const release = await createHandler()._lock('vm')
assert.equal(objects.get(KEY).lastModified.getTime(), providerDate().getTime())
await release()
})
it('keeps the lock when a refresh fails, and refreshes again', async () => {
const release = await createHandler()._lock('vm')
faults.putStatus = 500
mock.timers.tick(REFRESH_INTERVAL)
await flush()
assert.equal(objects.get(KEY).lastModified.getTime(), Date.now() - REFRESH_INTERVAL)
faults.putStatus = undefined
mock.timers.tick(REFRESH_INTERVAL)
await flush()
assert.equal(objects.get(KEY).lastModified.getTime(), Date.now())
await release()
assert.equal(objects.has(KEY), false)
})
it('waits for a refresh in flight before deleting the lock', async () => {
const release = await createHandler()._lock('vm')
let resolvePut
faults.pendingPut = new Promise(resolve => {
resolvePut = resolve
})
mock.timers.tick(REFRESH_INTERVAL)
const released = release()
await flush()
assert.equal(objects.has(KEY), true)
resolvePut()
await released
assert.equal(objects.has(KEY), false)
})
})

View File

@@ -1,18 +1,5 @@
# ChangeLog
## **next**
- [Pages] The page header and tabs now stay in place on every page: dashboards, System tabs, consoles, tasks, settings and creation forms scroll under them instead of scrolling with them (PR [#10822](https://github.com/vatesfr/xen-orchestra/pull/10822))
- [VM/Host] Main actions (shutdown, reboot, start, resume, migrate, snapshot) are shown as buttons in the VM and host headers and move one by one into "More actions" when space runs out; "More actions" groups them under "Change state" and "Actions" sections, and VMs without guest tools get force shutdown and force reboot instead (PR [#10815](https://github.com/vatesfr/xen-orchestra/pull/10815))
- [Web-Core/UiButtonIcon] Update icon buttons design, with a counter replacing the notification dot (PR [#10799](https://github.com/vatesfr/xen-orchestra/pull/10799))
- [Mobile/Side panels] Full-page menus on mobile with "Change state" in "More actions", side panel actions shown as buttons when there is room, and fixes for long names in headers and menu placement (PR [#10745](https://github.com/vatesfr/xen-orchestra/pull/10745))
- [Side panels] Side panels can be resized like the treeview, whose default width now adapts to the screen size (PR [#10725](https://github.com/vatesfr/xen-orchestra/pull/10725))
- [Side panels] Values stay next to their label when they fit and move below it otherwise, fields with several values (IP addresses, bond devices) get a copy button per value and a "Copy all" action, and rows, links and statuses are more evenly spaced and sized (PR [#10733](https://github.com/vatesfr/xen-orchestra/pull/10733))
- [Treeview] Fix translations of the tooltip for VM and host statuses in treeview (PR [#10643](https://github.com/vatesfr/xen-orchestra/pull/10643))
- [Web-Core/UiDropdownButton] Update dropdown buttons design (PR [#9850](https://github.com/vatesfr/xen-orchestra/pull/9850))
- [Tables] Tables are filtered as you type, half a second after the last keystroke, or right away with Enter; the "Search" button is removed (PR [#10797](https://github.com/vatesfr/xen-orchestra/pull/10797))
- [Side panels] Fix the double scrollbar when the side panel content is taller than the screen: the page header now stays in place, and the list and the side panel each scroll on their own. All scrollbars are now thin, like the side panel one (PR [#10807](https://github.com/vatesfr/xen-orchestra/pull/10807))
## **0.27.0** (2026-10-29)
- [New network] Choose between no NBD, secure NBD and insecure NBD (not encrypted) when creating a network (PR [#10540](https://github.com/vatesfr/xen-orchestra/pull/10540))

View File

@@ -94,8 +94,6 @@ useUnreachableHosts()
.main {
flex: 1;
overflow: auto;
display: flex;
flex-direction: column;
background-color: var(--color-neutral-background-secondary);
}
</style>

Some files were not shown because too many files have changed in this diff Show More