Compare commits

..

5 Commits

Author SHA1 Message Date
Thomas Moraine
c0266e5118 docs(xo): rebuild document hierarchy inside categories
Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>
2025-10-27 15:32:25 +01:00
Thomas Moraine
19e5ebc03f docs(xo): recreate document category hierarchy in the sidebar
Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>
2025-10-27 15:32:25 +01:00
Thomas Moraine
5d5f0fb2bd docs(xo): update links
Following the change in configuration from manual sidebar generation to automatic, documentation pages have been moved to category folder, which breaks internal links. This commit updates all the internal links to reflect the new file organization and prevent Docusaurus build failures

Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>
2025-10-27 15:32:06 +01:00
Thomas Moraine
c3606079a1 docs(xo): organize markdown documents into folders
Following the change in the sidebar configuration (from manual to automatic), we're moving the markdown documentation files to folders in order for Docusaurus to automatically re-generate the sidebar categories.

Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>
2025-10-27 14:58:12 +01:00
Thomas Moraine
bdb6842791 docs(sidebar.ts): switch sidebar generation from manual to automatic
This change in configuration makes it easier to maintain and update the documentation, and is the same configuration used in the XCP-ng documentation.

Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>
2025-10-27 14:57:58 +01:00
2278 changed files with 44199 additions and 151686 deletions

View File

@@ -233,29 +233,7 @@ module.exports = {
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
'@intlify/vue-i18n/no-missing-keys-in-other-locales': [
'error',
{
ignoreLocales: [
'de',
'fa',
'cs',
'es',
'sv',
'it',
'ru',
'uk',
'nl',
'pt-BR',
'da',
'ko',
'nb-NO',
'ja',
'fi',
'pl',
'zh-Hans',
'sk',
'pt',
],
},
{ ignoreLocales: ['de', 'fa', 'cs', 'es', 'sv', 'it', 'ru', 'uk', 'nl', 'pt_BR'] },
],
},
},
@@ -281,7 +259,7 @@ module.exports = {
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/src/route-map.d.ts'],
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
parserOptions: {
sourceType: 'module',
},
@@ -290,17 +268,6 @@ module.exports = {
'eslint-comments/no-unlimited-disable': 'off',
},
},
{
files: ['*.d.ts'],
parser: '@typescript-eslint/parser',
parserOptions: {
sourceType: 'module',
},
rules: {
'n/no-unsupported-features/es-syntax': 'off',
'no-unused-vars': 'off',
},
},
],
ignorePatterns: ['@vates/fatfs/'],

View File

@@ -25,7 +25,7 @@ jobs:
- name: Setup Node environment
uses: actions/setup-node@v4
with:
node-version: '>=22.12'
node-version: '20'
cache: 'yarn'
- name: Install project dependencies

View File

@@ -3,5 +3,4 @@
**/*.hbs
# Generated files
/@xen-orchestra/web/src/route-map.d.ts
/@xen-orchestra/lite/src/route-map.d.ts
/@xen-orchestra/web/typed-router.d.ts

View File

@@ -1,5 +0,0 @@
export function asyncEach<Item>(
iterable: Iterable<Item> | AsyncIterable<Item>,
iteratee: (item: Item, index: number, iterable: Iterable<Item> | AsyncIterable<Item>) => Promise<void>,
opts?: { concurrency?: number; signal?: AbortSignal; stopOnError?: boolean }
): Promise<void>

View File

@@ -67,17 +67,20 @@ describe('asyncEach', () => {
})
it('stops on first error when stopOnError is true', async () => {
const tracker = new assert.CallTracker()
const error = new Error()
const iteratee = spy((_, i) => {
const iteratee = tracker.calls((_, i) => {
if (i === 1) {
throw error
}
})
}, 2)
assert.deepStrictEqual(
await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true })),
error
)
assert.deepStrictEqual(iteratee.callCount, 2)
tracker.verify()
})
it('rejects AggregateError when stopOnError is false', async () => {
@@ -97,16 +100,19 @@ describe('asyncEach', () => {
})
it('can be interrupted with an AbortSignal', async () => {
const tracker = new assert.CallTracker()
const ac = new AbortController()
const iteratee = spy((_, i) => {
const iteratee = tracker.calls((_, i) => {
if (i === 1) {
ac.abort()
}
})
}, 2)
await assert.rejects(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal }), {
message: 'asyncEach aborted',
})
assert.deepStrictEqual(iteratee.callCount, 2)
tracker.verify()
})
it('handle error on iterable', async () => {

View File

@@ -24,8 +24,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"types": "index.d.ts",
"version": "1.0.3",
"version": "1.0.1",
"engines": {
"node": ">=8.10"
},

View File

@@ -24,8 +24,8 @@
},
"dependencies": {
"@vates/multi-key-map": "^0.2.0",
"@xen-orchestra/async-map": "^0.1.3",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.7.1",
"ensure-array": "^1.0.0"
},
"devDependencies": {

View File

@@ -1,12 +1,12 @@
import Fuse from 'fuse-native'
import LRU from 'lru-cache'
import Fuse from 'fuse-native'
import { createLogger } from '@xen-orchestra/log'
import { VhdSynthetic } from 'vhd-lib'
import { Disposable, fromCallback } from 'promise-toolbox'
const { warn } = createLogger('vates:fuse-vhd')
// build a stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
const stat = st => ({
mtime: st.mtime || new Date(),
atime: st.atime || new Date(),
@@ -20,109 +20,42 @@ const stat = st => ({
export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
const cache = new LRU({
max: 16, // each cached block is 2MB in size
})
await vhd.readBlockAllocationTable()
const { blockSize } = vhd.header
// A zeroed buffer returned for VHD blocks that are not present in the chain.
// Never mutated — safe to share across all reads.
const EMPTY_BLOCK = Buffer.alloc(blockSize, 0)
// --- coalescing block read queue ---
//
// FUSE read requests from NTFS-3g arrive in small chunks (typically 4 KiB)
// while VHD blocks are 2 MiB. Reading blocks one at a time from the backing
// store (CIFS/NFS/…) bounds libuv threadpool usage to 1 concurrent I/O call.
//
// Coalescing: all pending requests for the same blockId share a single
// readBlock() call. When the block resolves, every waiter is fulfilled at once.
//
// Together these two properties prevent the FUSE-on-FUSE threadpool deadlock:
// concurrent NTFS-3g reads no longer starve the VHD block reads they depend on.
// In-memory block cache. Each VHD block is 2 MiB; 16 blocks = 32 MiB.
// This is the primary tool against the 300× CIFS amplification: NTFS-3g
// reads a 2 MiB block in ~512 sequential 4 KiB FUSE requests. Without a
// cache, each request causes a full 2 MiB CIFS read. With the cache, only
// the first request reads from CIFS; the rest are served from RAM.
const blockCache = new LRU({ max: 64 })
const pendingBlocks = new Map() // blockId → Promise<Buffer>
const readQueue = [] // { blockId, resolve, reject }[]
let queueRunning = false
// log stats every 10 s while there is activity
function enqueueBlock(blockId) {
// 1. serve from cache if available
if (blockCache.has(blockId)) {
return Promise.resolve(blockCache.get(blockId))
}
// 2. coalesce: join an existing in-flight read for this block
if (pendingBlocks.has(blockId)) {
return pendingBlocks.get(blockId)
}
const p = new Promise((resolve, reject) => {
readQueue.push({ blockId, resolve, reject })
})
pendingBlocks.set(blockId, p)
p.then(
data => {
pendingBlocks.delete(blockId)
blockCache.set(blockId, data)
},
() => pendingBlocks.delete(blockId)
)
drainQueue()
return p
}
function drainQueue() {
if (queueRunning) return
queueRunning = true
// async IIFE — errors are forwarded to individual reject() callbacks,
// so the outer floating promise is always fulfilled
;(async () => {
while (readQueue.length > 0) {
const { blockId, resolve, reject } = readQueue.shift()
try {
resolve((await vhd.readBlock(blockId)).data)
} catch (err) {
reject(err)
}
}
queueRunning = false
})()
}
async function readData(buf, len, pos) {
if (len === 0) return 0
const startBlockId = Math.floor(pos / blockSize)
// use (pos + len - 1) so an exact block boundary doesn't include the next block
const endBlockId = Math.floor((pos + len - 1) / blockSize)
let copied = 0
for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
const data = vhd.containsBlock(blockId) ? await enqueueBlock(blockId) : EMPTY_BLOCK
const offsetStart = blockId === startBlockId ? pos % blockSize : 0
const offsetEnd = blockId === endBlockId ? ((pos + len - 1) % blockSize) + 1 : blockSize
data.copy(buf, copied, offsetStart, offsetEnd)
copied += offsetEnd - offsetStart
}
return copied
}
const fuse = new Fuse(mountDir, {
async readdir(path, cb) {
if (path === '/') return cb(null, ['vhd0'])
if (path === '/') {
return cb(null, ['vhd0'])
}
cb(Fuse.ENOENT)
},
async getattr(path, cb) {
if (path === '/') return cb(null, stat({ mode: 'dir', size: 4096 }))
if (path === '/vhd0') return cb(null, stat({ mode: 'file', size: vhd.footer.currentSize }))
if (path === '/') {
return cb(
null,
stat({
mode: 'dir',
size: 4096,
})
)
}
if (path === '/vhd0') {
return cb(
null,
stat({
mode: 'file',
size: vhd.footer.currentSize,
})
)
}
cb(Fuse.ENOENT)
},
read(path, fd, buf, len, pos, cb) {
if (path === '/vhd0') {
return readData(buf, len, pos).then(cb, error => {
return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
warn('read error', { path, len, pos, error })
cb(Fuse.EIO)
})
@@ -131,9 +64,7 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
},
})
return new Disposable(
() => {
return fromCallback(cb => fuse.unmount(cb))
},
() => fromCallback(cb => fuse.unmount(cb)),
fromCallback(cb => fuse.mount(cb))
)
})

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/fuse-vhd",
"version": "2.1.3",
"version": "2.1.2",
"license": "ISC",
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,12 +19,12 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.9.3",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/fs": "^4.6.4",
"@xen-orchestra/log": "^0.7.1",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.16.1"
"vhd-lib": "^4.14.2"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/generator-toolbox",
"version": "1.1.3",
"version": "1.1.0",
"main": "dist/index.mjs",
"license": "MIT",
"private": false,
@@ -10,7 +10,7 @@
"rimraf": "^6.0.1",
"@types/node": "^20.6",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0"
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",

View File

@@ -144,7 +144,14 @@ class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
return this.#parent.remove(this.#uid, e)
}
[Symbol.asyncIterator](): AsyncGenerator<T, TReturn, TNext> {
return this
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
while (true) {
const res = await this.next()
if (res.done) {
break
}
yield res.value
}
}
}

View File

@@ -2,7 +2,8 @@ import assert from 'node:assert'
import { suite, test } from 'node:test'
import { Synchronized } from './synchronized.mjs'
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (_val: unknown) => {}) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (val: unknown) => {}) {
for (let i = 0; i < end; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
@@ -16,7 +17,7 @@ async function consume(
iterable: AsyncGenerator,
delay = 2500,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
onConsumed = (_val: unknown, _iterable: AsyncGenerator) => Promise.resolve(false)
onConsumed = (val: unknown, iterable: AsyncGenerator) => Promise.resolve(false)
) {
for await (const val of iterable) {
await new Promise(resolve => setTimeout(resolve, delay))
@@ -27,30 +28,6 @@ async function consume(
}
suite('success', () => {
test('for-await-of break stops the source generator', async () => {
let sourceClosed = false
const source = (async function* () {
try {
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
}
} finally {
sourceClosed = true
}
})()
const forker = new Synchronized(source)
const fork = forker.fork('first')
// for-await-of must call fork.return() on break, which propagates to the source
for await (const val of fork) {
if (val === 2) break
}
assert.strictEqual(sourceClosed, true)
})
test('if works with multiple consumer', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(3, progress)

View File

@@ -1,7 +1,4 @@
import assert from 'node:assert'
const TIMEOUT = Symbol('timeout')
export class Timeout<T> implements AsyncGenerator {
#source: AsyncGenerator<T>
#timeout: number
@@ -11,30 +8,21 @@ export class Timeout<T> implements AsyncGenerator {
this.#timeout = timeout
}
async next(): Promise<IteratorResult<T>> {
let timeoutHandle: ReturnType<typeof setTimeout>
const sourceNext = this.#source.next()
const result = await Promise.race([
sourceNext.then(res => {
clearTimeout(timeoutHandle)
return res
}),
new Promise<typeof TIMEOUT>(resolve => {
timeoutHandle = setTimeout(() => resolve(TIMEOUT), this.#timeout)
}),
])
if (result === TIMEOUT) {
// stop the source once the in-flight next settles to avoid data loss on the next call
// we don't want to await for the end of timeouted code
sourceNext.then(
() => this.#source.return(undefined).catch(() => {}),
() => {} // source already errored, nothing to clean up
)
throw new Error('Timeout reached')
}
return result
let timeout: ReturnType<typeof setTimeout>
const promiseTimeout = new Promise((_, reject) => {
timeout = setTimeout(() => {
reject(new Error('Timeout reached '))
}, this.#timeout)
})
const promiseNext = new Promise((resolve, reject) => {
this.#source.next().then(res => {
// ensure timetout won't fire later
clearTimeout(timeout)
resolve(res)
}, reject)
})
// promiseTimeout will never resolve
return Promise.race([promiseNext, promiseTimeout]) as Promise<IteratorResult<T>>
}
return(): Promise<IteratorResult<T>> {
return this.#source.return(undefined)

View File

@@ -64,24 +64,4 @@ describe('Timeout class', () => {
await assert.rejects(timeout.throw(new Error('Test error')), 'Expected error was not thrown')
})
it('should close the source generator after a timeout', async () => {
let sourceClosed = false
const slowSource = (async function* () {
try {
// resolves after 100ms so the in-flight next() eventually settles
await new Promise(resolve => setTimeout(resolve, 100))
yield 1
} finally {
sourceClosed = true
}
})()
const timeout = new Timeout(slowSource, 10) // fires before source yields
await assert.rejects(() => timeout.next(), /Timeout reached/)
// wait for the in-flight next() to settle so source.return() can be called
await new Promise(resolve => setTimeout(resolve, 200))
assert.strictEqual(sourceClosed, true)
})
})

View File

@@ -1,65 +0,0 @@
### `create([opts], [requestListener])`
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
```js
import { create } from '@vates/http-server-plus'
const server = create(function onRequest(req, res) {
res.end('Hello')
})
```
### `server.listen(opts)`
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
```js
// HTTP on a specific port
await server.listen({ port: 8080 })
// HTTPS — detected automatically when cert+key are provided
await server.listen({
port: 443,
cert: fs.readFileSync('certificate.pem'),
key: fs.readFileSync('private_key.pem'),
})
// Unix socket
await server.listen({ socket: '/var/run/http.sock' })
// File descriptor
await server.listen({ fd: 3 })
// Systemd socket activation
await server.listen({ systemdSocket: 0 })
```
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
Multiple endpoints can be started on the same instance:
```js
const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
console.log('listening on', httpAddr, httpsAddr)
```
### HTTP/2
Pass a custom `createSecureServer` factory to enable HTTP/2:
```js
import { createSecureServer } from 'node:http2'
import { create } from '@vates/http-server-plus'
const server = create({ createSecureServer }, app)
await server.listen({ port: 443, cert, key })
```
### `server.close()`
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
```js
await server.close()
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,98 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/http-server-plus
[![Package Version](https://badgen.net/npm/v/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus) ![License](https://badgen.net/npm/license/@vates/http-server-plus) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/http-server-plus)](https://bundlephobia.com/result?p=@vates/http-server-plus) [![Node compatibility](https://badgen.net/npm/node/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus)
> Augmented &#x60;http.Server&#x60;, HTTP/HTTPS/HTTP2 and multiple ports on the same instance
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/http-server-plus):
```sh
npm install --save @vates/http-server-plus
```
## Usage
### `create([opts], [requestListener])`
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
```js
import { create } from '@vates/http-server-plus'
const server = create(function onRequest(req, res) {
res.end('Hello')
})
```
### `server.listen(opts)`
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
```js
// HTTP on a specific port
await server.listen({ port: 8080 })
// HTTPS — detected automatically when cert+key are provided
await server.listen({
port: 443,
cert: fs.readFileSync('certificate.pem'),
key: fs.readFileSync('private_key.pem'),
})
// Unix socket
await server.listen({ socket: '/var/run/http.sock' })
// File descriptor
await server.listen({ fd: 3 })
// Systemd socket activation
await server.listen({ systemdSocket: 0 })
```
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
Multiple endpoints can be started on the same instance:
```js
const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
console.log('listening on', httpAddr, httpsAddr)
```
### HTTP/2
Pass a custom `createSecureServer` factory to enable HTTP/2:
```js
import { createSecureServer } from 'node:http2'
import { create } from '@vates/http-server-plus'
const server = create({ createSecureServer }, app)
await server.listen({ port: 443, cert, key })
```
### `server.close()`
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
```js
await server.close()
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,268 +0,0 @@
'use strict'
// ===================================================================
const EventEmitter = require('events').EventEmitter
const formatUrl = require('url').format
const fromEvent = require('promise-toolbox/fromEvent')
const inherits = require('util').inherits
const resolvePath = require('path').resolve
// ===================================================================
const forwardedEvents = [
'checkContinue',
'checkExpectation',
'clientError',
'connect',
'connection',
'listening',
'request',
'secureConnection',
'upgrade',
]
function extractProperty(obj, prop) {
const value = obj[prop]
if (value !== undefined) {
delete obj[prop]
return value
}
}
function forOwn(obj, cb) {
const keys = Object.keys(obj)
for (let i = 0, n = keys.length; i < n; ++i) {
cb(obj[keys[i]])
}
}
function getSystemdFd(index) {
if (process.pid !== +process.env.LISTEN_PID) {
throw new Error('systemd sockets are not meant for us (LISTEN_PID)')
}
const listenFds = process.env.LISTEN_FDS
if (listenFds == null) {
throw new Error('no systemd sockets found (LISTEN_FDS)')
}
const max = listenFds - 1
if (index > max) {
throw new Error('no such systemd socket found')
}
return index + 3 // 3 = SD_LISTEN_FDS_START
}
const isEmpty = obj => Object.keys(obj).length === 0
// ===================================================================
function onNewListener(event, listener) {
if (forwardedEvents.includes(event)) {
forOwn(this._servers, server => {
server.on(event, listener)
})
}
}
function onRemoveListener(event, listener) {
if (forwardedEvents.includes(event)) {
forOwn(this._servers, server => {
server.off(event, listener)
})
}
}
function Server(opts) {
EventEmitter.call(this)
this._createServer = (opts != null && opts.createServer) || require('http').createServer
this._createSecureServer = (opts != null && opts.createSecureServer) || require('https').createServer
this._servers = Object.create(null)
this._niceAddresses = Object.create(null)
this.on('removeListener', onRemoveListener).on('newListener', onNewListener)
}
inherits(Server, EventEmitter)
const proto = Server.prototype
function getAddress(serverId) {
return this[serverId].address()
}
proto.addresses = function Server$addresses() {
const servers = this._servers
return Object.keys(servers).map(getAddress, servers)
}
function getNiceAddress(serverId) {
return this[serverId]
}
proto.niceAddresses = function Server$niceAddresses() {
const niceAddresses = this._niceAddresses
return Object.keys(niceAddresses).map(getNiceAddress, niceAddresses)
}
function close(server) {
server.close()
}
proto.close = function Server$close(callback) {
if (callback) {
this.once('close', callback)
}
// Emit the close event even if there are no registered servers.
if (isEmpty(this._servers)) {
const self = this
setImmediate(function () {
self.emit('close')
})
return Promise.resolve()
}
// Closes each servers.
forOwn(this._servers, close)
return fromEvent(this, 'close')
}
let nextId = 0
proto.listen = function Server$listen(opts) {
opts = Object.assign({}, opts)
if (opts.hostname === 'localhost') {
return Promise.all([
this.listen(Object.assign(opts, { hostname: '::1' })),
this.listen(Object.assign(opts, { hostname: '127.0.0.1' })),
]).then(addresses => addresses[0])
}
const hostname = extractProperty(opts, 'hostname')
let port = extractProperty(opts, 'port')
const systemdSocket = extractProperty(opts, 'systemdSocket')
let fd = extractProperty(opts, 'fd')
let socket = extractProperty(opts, 'socket')
const servers = this._servers
const niceAddresses = this._niceAddresses
let server, protocol
if (opts.pfx || opts.SNICallback || (opts.cert && opts.key)) {
server = this._createSecureServer(opts)
protocol = 'https'
} else {
server = this._createServer(opts)
protocol = 'http'
}
const id = nextId++
servers[id] = server
if (systemdSocket != null) {
fd = getSystemdFd(systemdSocket)
}
// Compute a temporary nice address to display in case of error.
let niceAddress
if (fd != null) {
server.listen({ fd })
niceAddress = protocol + '://<fd:' + fd + '>'
} else if (socket != null) {
socket = resolvePath(socket)
server.listen(socket)
niceAddress = protocol + '://' + socket
} else {
if (port == null) {
port = protocol === 'https' ? 443 : 80
}
server.listen(port, hostname)
niceAddress = formatUrl({
protocol,
// Hostname default to localhost.
hostname: hostname || 'localhost',
// No port means random, unknown for now.
port: port || '<unknown>',
})
}
const emit = this.emit.bind(this)
server.once('close', function onClose() {
delete servers[id]
delete niceAddresses[id]
if (isEmpty(servers)) {
emit('close')
}
})
server.on('error', function onError() {
delete servers[id]
// FIXME: Should it be forwarded and be fatal if there are no
// listeners?
})
forwardedEvents.forEach(event => {
this.listeners(event).forEach(listener => {
server.on(event, listener)
})
})
return fromEvent(server, 'listening').then(
function () {
const address = server.address()
const niceAddress =
typeof address === 'string'
? protocol + '://' + address
: formatUrl({
protocol,
hostname: address.address,
port: address.port,
})
niceAddresses[id] = niceAddress
return niceAddress
},
function (error) {
error.niceAddress = niceAddress
throw error
}
)
}
function ref(server) {
server.ref()
}
proto.ref = function Server$ref() {
forOwn(this._servers, ref)
}
function unref(server) {
server.unref()
}
proto.unref = function Server$unref() {
forOwn(this._servers, unref)
}
// ===================================================================
module.exports = exports = Server
exports.create = function create(opts, requestListener) {
if (typeof opts === 'function') {
requestListener = opts
opts = undefined
}
const server = new Server(opts)
if (requestListener !== undefined) {
server.on('request', requestListener)
}
return server
}

View File

@@ -1,303 +0,0 @@
'use strict'
// ===================================================================
const assert = require('assert/strict')
const { EventEmitter } = require('events')
const resolvePath = require('path').resolve
const { describe, it } = require('node:test')
const HttpServerPlus = require('./')
// ===================================================================
// Creates a minimal mock server that can be used as a drop-in for http/https.Server.
// `address` is the value returned by mock.address() — use an object for TCP, a string for sockets.
function createMock(address) {
const mock = new EventEmitter()
mock.listen = function () {
setImmediate(() => this.emit('listening'))
}
mock.address = () => address
mock.close = function () {
setImmediate(() => this.emit('close'))
}
return mock
}
// ===================================================================
describe('HttpServerPlus', function () {
describe('#create()', function () {
it('creates a new instance', function () {
assert(HttpServerPlus.create() instanceof HttpServerPlus)
})
it('can register a `request` listener', function () {
const listener = function () {}
const server = HttpServerPlus.create(listener)
assert.deepEqual(server.listeners('request'), [listener])
})
})
describe('.addresses()', function () {
it('returns the list of addresses the server is listening at', function () {
const addr = { address: '127.0.0.1', port: 8080, family: 'IPv4' }
const mock = createMock(addr)
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(function () {
assert.deepEqual(server.addresses(), [addr])
return server.close()
})
})
})
describe('.niceAddresses()', function () {
it('returns the list of human readable addresses the server is listening at', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function (address) {
assert.deepEqual(server.niceAddresses(), [address])
return server.close()
})
})
})
describe('secureConnection event forwarding', function () {
function createMockHttpsServer() {
const server = new EventEmitter()
server.listen = function () {
setImmediate(() => this.emit('listening'))
}
server.address = () => ({ address: '127.0.0.1', port: 8443 })
server.close = function () {
setImmediate(() => this.emit('close'))
}
return server
}
function createServerWithMock() {
let mockServer
const server = HttpServerPlus.create({
createSecureServer: () => {
mockServer = createMockHttpsServer()
return mockServer
},
})
return {
server,
getMock: () => mockServer,
}
}
it('forwards secureConnection to listeners registered before listen()', function () {
const { server, getMock } = createServerWithMock()
const received = []
server.on('secureConnection', socket => received.push(socket))
const fakeSocket = {}
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
getMock().emit('secureConnection', fakeSocket)
assert.deepEqual(received, [fakeSocket])
return server.close()
})
})
it('forwards secureConnection to listeners registered after listen()', function () {
const { server, getMock } = createServerWithMock()
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
const received = []
server.on('secureConnection', socket => received.push(socket))
const fakeSocket = {}
getMock().emit('secureConnection', fakeSocket)
assert.deepEqual(received, [fakeSocket])
return server.close()
})
})
it('stops forwarding secureConnection after listener is removed', function () {
const { server, getMock } = createServerWithMock()
const received = []
const listener = socket => received.push(socket)
server.on('secureConnection', listener)
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
server.off('secureConnection', listener)
getMock().emit('secureConnection', {})
assert.deepEqual(received, [])
return server.close()
})
})
})
describe('.listen()', function () {
it('can use a host:port', function () {
let listenArgs
const mock = createMock({ address: '127.0.0.1', port: 8080 })
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function () {
assert.deepEqual(listenArgs, [8080, '127.0.0.1'])
return server.close()
})
})
it('can use a systemd socket', function () {
const SD_LISTEN_FDS_START = 3
const SOCKET_INDEX = 0
let listenArgs
const mock = createMock('http://<fd:3>')
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
const env = process.env
env.LISTEN_FDS = String(SOCKET_INDEX + 1)
env.LISTEN_PID = String(process.pid)
return server.listen({ systemdSocket: SOCKET_INDEX }).then(function () {
delete env.LISTEN_FDS
delete env.LISTEN_PID
assert.deepEqual(listenArgs, [{ fd: SOCKET_INDEX + SD_LISTEN_FDS_START }])
return server.close()
})
})
it('can use a socket', function () {
const socketPath = '/tmp/http-server-plus-test.sock'
let listenArgs
const mock = createMock(socketPath)
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ socket: socketPath }).then(function () {
assert.deepEqual(listenArgs, [resolvePath(socketPath)])
return server.close()
})
})
it('can use a HTTPS certificate', function () {
let capturedOpts
const mock = createMock({ address: '127.0.0.1', port: 8443 })
const server = HttpServerPlus.create({
createSecureServer: opts => {
capturedOpts = opts
return mock
},
})
return server.listen({ cert: 'my-cert', key: 'my-key', port: 8443 }).then(function (address) {
assert.equal(capturedOpts.cert, 'my-cert')
assert.equal(capturedOpts.key, 'my-key')
assert.ok(address.startsWith('https://'), `expected https URL, got: ${address}`)
return server.close()
})
})
it('returns a promise which will resolve once listening', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(function (address) {
assert.equal(typeof address, 'string')
assert.ok(address.startsWith('http://'), `expected http URL, got: ${address}`)
return server.close()
})
})
it('returns a promise which will reject if failure', function () {
const listenError = new Error('EADDRINUSE')
const mock = createMock(null)
mock.listen = function () {
setImmediate(() => this.emit('error', listenError))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(
function () {
assert.fail('should have rejected')
},
function (err) {
assert.equal(err, listenError)
assert.equal(typeof err.niceAddress, 'string')
}
)
})
})
describe('.close()', function () {
it('closes all servers', function () {
const closed = []
let callCount = 0
const server = HttpServerPlus.create({
createServer: () => {
const id = callCount++
const mock = createMock({ address: '127.0.0.1', port: id })
mock.close = function () {
closed.push(id)
setImmediate(() => this.emit('close'))
}
return mock
},
})
return Promise.all([server.listen({ port: 0 }), server.listen({ port: 0 })])
.then(function () {
return server.close()
})
.then(function () {
assert.equal(closed.length, 2)
})
})
it('emit the `close` event when all servers are closed', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
let closeEmitted = false
server.on('close', () => {
closeEmitted = true
})
return server
.listen({ port: 8080 })
.then(function () {
return server.close()
})
.then(function () {
assert.ok(closeEmitted)
})
})
it('emit the `close` event even if there are no servers', function () {
const server = HttpServerPlus.create()
// close() with no servers returns Promise.resolve() immediately (before emitting
// 'close' via setImmediate), so we must wait for the event itself, not the promise.
return new Promise(function (resolve) {
server.close(resolve)
})
})
})
})

View File

@@ -1,38 +0,0 @@
{
"name": "@vates/http-server-plus",
"version": "2.0.2",
"license": "ISC",
"description": "Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance",
"keywords": [
"http",
"https",
"server",
"ports"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/http-server-plus",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/http-server-plus",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20"
},
"dependencies": {
"promise-toolbox": "^0.21.0"
},
"devDependencies": {
"eslint-plugin-n": "^15.1.0"
},
"scripts": {
"pretest": "eslint .",
"test": "node --test index.spec.js",
"postversion": "npm publish --access public"
},
"private": false,
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
}
}

View File

@@ -19,12 +19,6 @@ export class NbdDisk extends RandomAccessDisk {
/** @type {number} */
#blockSize
/** @type {number | undefined} */
#hasBlockCursor
/** @type {number | undefined} */
#hasBlockPreviousIndex
constructor(nbdInfos, blockSize, { dataMap } = {}) {
super()
this.#blockSize = blockSize
@@ -33,38 +27,11 @@ export class NbdDisk extends RandomAccessDisk {
}
#processDatamap(rawDataMap) {
const ranges = rawDataMap
.filter(({ type, length }) => type === 0 && length > 0)
return rawDataMap
.filter(({ type }) => type === 0)
.map(({ offset, length }) => ({ offset, length }))
.sort(({ offset: offset1 }, { offset: offset2 }) => offset1 - offset2)
// hasBlock()'s forward-only cursor is only correct if the extents are
// sorted AND disjoint: an earlier extent must never reach into a block
// after a later one, otherwise the cursor could skip it and wrongly report
// a block as empty, dropping data and corrupting the output.
// Touching extents (cur.offset === prevEnd) are merged
const merged = []
for (const range of ranges) {
const last = merged[merged.length - 1]
if (last !== undefined) {
const lastEnd = last.offset + last.length
if (range.offset < lastEnd) {
throw new Error(
`overlapping ranges in data map: [${last.offset}, ${lastEnd}) and [${range.offset}, ${range.offset + range.length})`
)
}
if (range.offset === lastEnd) {
// touching: extend the previous extent
last.length += range.length
continue
}
}
// gap (or first range): new extent
merged.push(range)
}
return merged
}
/**
* @param {number} index
* @returns {Promise<DiskBlock>}
@@ -73,17 +40,7 @@ export class NbdDisk extends RandomAccessDisk {
if (!this.#nbdClient) {
throw new Error("can't readBlock before init")
}
let data = await this.#nbdClient.readBlock(index, this.getBlockSize())
if (data.length !== this.getBlockSize()) {
// this condition only work on non aligned disks
// we won't accept truncated block from unaligned disks
if (index === Math.ceil(this.getVirtualSize() / this.getBlockSize()) - 1) {
// last block add zeros at the end
data = Buffer.concat([data], this.getBlockSize())
} else {
throw new Error(`Block ${index} is not at the right size, expecting ${this.getBlockSize()}, got ${data.length}`)
}
}
const data = await this.#nbdClient.readBlock(index, this.getBlockSize())
return {
index,
data,
@@ -150,34 +107,6 @@ export class NbdDisk extends RandomAccessDisk {
return [...indexes].sort((a, b) => a - b)
}
/**
* Counts the allocated blocks without materializing the index list.
*
* @returns {number}
*/
getBlockIndexesCount() {
if (!this.#dataMap) {
throw new Error("can't getBlockIndexesCount before init")
}
const blockSize = this.getBlockSize()
let count = 0
let lastCountedBlock = -1
for (const { offset, length } of this.#dataMap) {
const firstBlockIndex = Math.floor(offset / blockSize)
const lastBlockIndex = Math.floor((offset + length - 1) / blockSize)
const from = Math.max(firstBlockIndex, lastCountedBlock + 1)
if (lastBlockIndex >= from) {
count += lastBlockIndex - from + 1
lastCountedBlock = lastBlockIndex
}
}
return count
}
/**
* @param {number} index
* @returns {boolean}
@@ -186,32 +115,8 @@ export class NbdDisk extends RandomAccessDisk {
if (!this.#dataMap) {
throw new Error("can't hasBlock before init")
}
const blockSize = this.getBlockSize()
const blockStart = index * blockSize
const blockEnd = blockStart + blockSize
let startExtentIndex = 0
if (this.#hasBlockCursor !== undefined && index >= this.#hasBlockPreviousIndex) {
startExtentIndex = this.#hasBlockCursor
} else {
this.#hasBlockCursor = undefined
}
this.#hasBlockPreviousIndex = index
const dataMap = this.#dataMap
const l = dataMap.length
for (let i = startExtentIndex; i < l; i++) {
const { offset, length } = dataMap[i]
if (offset >= blockEnd) {
// extents are sorted: nothing from here on can overlap this block
break
}
if (offset + length > blockStart) {
this.#hasBlockCursor = i
return true
}
}
return false
const blockStart = index * this.getBlockSize()
const blockEnd = (index + 1) * this.getBlockSize()
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && blockEnd >= offset)
}
}

View File

@@ -1,127 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert'
import { NbdDisk } from './NbdDisk.mjs'
const KiB = 1024
const MiB = 1024 * KiB
const TiB = 1024 * 1024 * MiB
const SOURCE_BLOCK_SIZE = 2 * MiB // VHD_BLOCK_SIZE, the block size used when importing from VMware
const CLUSTER_SIZE = 64 * KiB // qcow2 cluster size
// hasBlock() only needs the dataMap (passed through the constructor) and the
// block size, so we can exercise it without a live NBD connection.
describe('NbdDisk.hasBlock', () => {
it('reports allocated and unallocated blocks correctly', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(1), false)
assert.strictEqual(disk.hasBlock(4), false)
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(6), false)
})
it('handles repeated and out-of-order queries with the resume cursor', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
// repeated indexes (the qcow2 path queries each 2MB block ~32 times as it
// walks 64KB clusters) must return a stable answer
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(6), false)
assert.strictEqual(disk.hasBlock(6), false)
// querying backwards must still be correct (falls back to a full scan)
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(5), true)
})
it('invalidates the cursor on a backward miss (no stale skip)', () => {
const dataMap = [
{ type: 0, offset: 3 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 3
{ type: 0, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 10
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(10), true) // forward hit: cursor parks on the high extent
assert.strictEqual(disk.hasBlock(1), false) // backward miss: must invalidate the cursor
assert.strictEqual(disk.hasBlock(3), true) // lower extent must not be skipped
})
it('throws on overlapping ranges (would break the resume cursor)', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE },
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE }, // overlaps the previous range
]
assert.throws(() => new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap }), /overlapping ranges/)
})
it('merges touching ranges and drops zero-length / non type-0 entries', () => {
const dataMap = [
{ type: 0, offset: 2 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [2, 3)
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5)
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [1, 2)
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 }, // [.5, 1), touches [0, .5)
{ type: 1, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // wrong type: ignored
{ type: 0, offset: 7 * SOURCE_BLOCK_SIZE, length: 0 }, // zero length: ignored
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(1), true)
assert.strictEqual(disk.hasBlock(2), true)
assert.strictEqual(disk.hasBlock(3), false)
assert.strictEqual(disk.hasBlock(7), false) // zero-length range dropped
assert.strictEqual(disk.hasBlock(10), false) // non type-0 range dropped
})
})
describe('NbdDisk.getBlockIndexesCount', () => {
const cases = {
'aligned, disjoint blocks': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
],
'a range spanning several blocks': [
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: 3 * SOURCE_BLOCK_SIZE }, // blocks 0,1,2,3
],
'two disjoint extents sharing block 0 (needs dedup)': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5) => block 0
{ type: 0, offset: 3 * (SOURCE_BLOCK_SIZE / 4), length: SOURCE_BLOCK_SIZE }, // [.75, 1.75) => blocks 0,1
],
'touching sub-block fragments': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 },
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 },
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE },
],
empty: [],
}
for (const [name, dataMap] of Object.entries(cases)) {
it(`matches getBlockIndexes().length: ${name}`, () => {
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
})
}
it('matches getBlockIndexes().length on the large sparse map', () => {
const virtualSize = Math.floor(3.5 * TiB)
const NB_RANGES = 113528
const dataMap = []
for (let i = 0; i < NB_RANGES; i++) {
dataMap.push({ type: 0, offset: Math.floor((i / NB_RANGES) * virtualSize), length: CLUSTER_SIZE })
}
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
})
})

View File

@@ -353,7 +353,6 @@ export default class NbdClient {
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
return pRetry(() => this.#readBlock(index, size), {
tries: this.#readBlockRetries,
when: error => error.code !== 'ERR_ABORTED',
onRetry: async err => {
warn('will retry reading block ', index, err)
await this.reconnect()
@@ -369,9 +368,9 @@ export default class NbdClient {
* A promise that resolves to an array where each object represents a segment:
* - `offset` — The byte offset from the start.
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type ( 0 means data).
* - `type` — A numeric code indicating the segment type ( 0 means no data).
*/
/* async */ getMap(signal) {
/* async */ getMap() {
return new Promise((resolve, reject) => {
const process = spawn('nbdinfo', [
'--json',
@@ -382,17 +381,8 @@ export default class NbdClient {
let errText = ''
process.stdout.on('data', data => (text += data))
process.stderr.on('data', data => (errText += data))
const onAbort = () => {
process.kill()
reject(signal.reason)
}
signal?.addEventListener('abort', onAbort, { once: true })
process.on('error', err => {
signal?.removeEventListener('abort', onAbort)
reject(err)
})
process.on('error', reject)
process.on('close', code => {
signal?.removeEventListener('abort', onAbort)
if (code !== 0) {
return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
}

View File

@@ -138,8 +138,8 @@ export default class MultiNbdClient {
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type (0 means no data).
*/
async getMap(signal) {
async getMap() {
// ask the map from one of the connected client
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap(signal)
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap()
}
}

View File

@@ -13,19 +13,19 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.5.0",
"version": "3.2.1",
"engines": {
"node": ">=14.0"
},
"main": "./index.mjs",
"dependencies": {
"@vates/async-each": "^1.0.3",
"@vates/async-each": "^1.0.1",
"@vates/read-chunk": "^1.2.1",
"@xen-orchestra/async-map": "^0.1.3",
"@xen-orchestra/disk-transform": "^1.3.3",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/disk-transform": "^1.2.0",
"@xen-orchestra/log": "^0.7.1",
"promise-toolbox": "^0.21.0",
"xen-api": "^4.8.0"
"xen-api": "^4.7.4"
},
"devDependencies": {
"tap": "^18.7.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/node-vsphere-soap",
"version": "2.1.3",
"version": "2.1.2",
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
"main": "lib/client.mjs",
"author": "reedog117",
@@ -11,8 +11,8 @@
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/log": "^0.7.2",
"axios": "^1.15.0",
"@xen-orchestra/log": "^0.7.1",
"axios": "^1.8.0",
"soap": "^1.0.0",
"soap-cookie": "^0.10.1"
},

View File

@@ -23,7 +23,7 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
Object.defineProperty(taskLog, '$root', { value: taskLog })
// start of a root task
onRootTaskStart(taskLog, event)
onRootTaskStart(taskLog)
} else {
// start of a subtask
const parent = taskLogs.get(parentId)
@@ -54,10 +54,10 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
}
if (type === 'end' && taskLog.$root === taskLog) {
onRootTaskEnd(taskLog, event)
onRootTaskEnd(taskLog)
}
}
onTaskUpdate(taskLog, event)
onTaskUpdate(taskLog)
}
}

View File

@@ -7,27 +7,6 @@ function alreadyEnded() {
throw new Error('task has already ended')
}
const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
// Create a serializable object from an error.
//
// Otherwise some fields might be non-enumerable and missing from logs.
const serializeError = error => {
return error instanceof Error
? {
...error, // Copy enumerable properties.
cause: error.cause,
code: error.code,
errors: serializeErrors(error.errors), // supports AggregateError
message: error.message,
name: error.name,
stack: error.stack,
}
: error
}
exports.serializeError = serializeError
// define a read-only, non-enumerable, non-configurable property
function define(object, property, value) {
Object.defineProperty(object, property, { value })
@@ -93,11 +72,7 @@ class Task {
_onProgress
get id() {
// Use a compact, sortable, string representation of the creation date,
// and add a random part to differentiate tasks created at the same time
//
// Due to the padding, dates are sortable up to 5188-04-22T11:04:28.415Z
return (this.id = Date.now().toString(36).padStart(9, '0') + '-' + Math.random().toString(36).slice(2))
return (this.id = Math.random().toString(36).slice(2))
}
set id(value) {
define(this, 'id', value)
@@ -161,10 +136,6 @@ class Task {
assert.equal(this.#status, PENDING)
assert.equal(this.#running, false)
if (result instanceof Error && result.toJSON === undefined) {
result = serializeError(result)
}
this.#emit('end', { status, result })
this._onProgress = alreadyEnded
this.#status = status

View File

@@ -4,7 +4,7 @@ const assert = require('node:assert').strict
const { describe, it } = require('node:test')
const { makeOnProgress } = require('./combineEvents.js')
const { Task, serializeError } = require('./index.js')
const { Task } = require('./index.js')
const noop = Function.prototype
@@ -28,36 +28,6 @@ function createTask(opts) {
return task
}
describe('serializeError', function () {
it('serializes simple error', function () {
const err = new Error('test Error')
err.code = 404
const serialized = serializeError(err)
assert.equal(serialized.message, 'test Error')
assert.equal(serialized.name, 'Error')
assert.equal(serialized.code, 404)
assert.notEqual(serialized.stack, undefined)
})
it('serializes aggregate error', function () {
const err = new AggregateError([new Error('test error 1'), new Error('test error 2')], 'Error message', {
cause: 'broken',
})
const serialized = serializeError(err)
assert.equal(serialized.message, 'Error message')
assert.equal(serialized.name, 'AggregateError')
assert.equal(serialized.cause, 'broken')
assert.notEqual(serialized.stack, undefined)
assert.equal(serialized.errors.length, 2)
assert.equal(serialized.errors[0].name, 'Error')
assert.equal(serialized.errors[1].name, 'Error')
assert.equal(serialized.errors[0].message, 'test error 1')
assert.equal(serialized.errors[1].message, 'test error 2')
assert.notEqual(serialized.errors[0].stack, undefined)
assert.notEqual(serialized.errors[1].stack, undefined)
})
})
describe('Task', function () {
describe('constructor', function () {
it('data properties are passed to the start event', async function () {
@@ -136,7 +106,7 @@ describe('Task', function () {
assert.equal(task.$events.length, 3)
assertEvent(task, { type: 'start' }, 0)
assertEvent(task, { type: 'abortionRequested', reason }, 1)
assertEvent(task, { type: 'end', status: 'failure', result: serializeError(result) }, 2)
assertEvent(task, { type: 'end', status: 'failure', result }, 2)
})
it('does not abort if the task succeed', async function () {
@@ -206,7 +176,7 @@ describe('Task', function () {
assertEvent(task, {
status: 'failure',
result: serializeError(error),
result: error,
type: 'end',
})
})
@@ -505,7 +475,7 @@ describe('Task', function () {
assert.equal(task.status, 'failure')
assertEvent(task, {
status: 'failure',
result: serializeError(e),
result: e,
type: 'end',
})
})
@@ -541,7 +511,7 @@ describe('Task', function () {
assert.equal(task.status, 'failure')
assertEvent(task, {
status: 'failure',
result: serializeError(e),
result: e,
type: 'end',
})
})

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.7.0",
"version": "0.6.2",
"engines": {
"node": ">=14"
},

View File

@@ -1,7 +1,7 @@
{
"name": "@vates/types",
"private": false,
"version": "1.30.1",
"version": "1.13.0",
"main": "./dist/index.mjs",
"exports": {
".": {
@@ -9,7 +9,6 @@
"types": "./dist/index.d.mts"
},
"./xo": "./dist/xo.d.mts",
"./xo-app": "./dist/xo-app.d.mts",
"./xen-api": "./dist/xen-api.d.mts",
"./common": {
"default": "./dist/common.mjs",
@@ -32,8 +31,7 @@
"devDependencies": {
"@eslint/js": "^9.19.0",
"typescript": "~5.6.3",
"typescript-eslint": "^8.61.0",
"@types/express": "^5.0.0"
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",

View File

@@ -1,4 +1,4 @@
import type { XoHost, XoNetwork, XoVif } from './xo.mjs'
import type { XoHost } from './xo.mjs'
declare const __brand: unique symbol
@@ -740,32 +740,3 @@ export type XapiVmStatsRaw = {
export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>
export type TrafficRuleProtocol = (typeof TRAFFIC_RULE_PROTOCOLS)[number]
export const SDN_CONTROLLER_OF_RULES_KEY = 'xo:sdn-controller:of-rules'
export const SDN_CONTROLLER_OF_METHOD_KEY = 'xo:sdn-controller:of-method'
export const SDN_CONTROLLER_OF_FORMAT_KEY = 'xo:sdn-controller:of-format'
export const TRAFFIC_RULE_PROTOCOLS = ['ARP', 'ICMP', 'IP', 'TCP', 'UDP'] as const
export const TRAFFIC_RULE_PROTOCOLS_WITH_PORT: readonly TrafficRuleProtocol[] = ['TCP', 'UDP']
export type TrafficRuleTargetType = XoNetwork['type'] | XoVif['type']
export type TrafficRuleDirection = 'from' | 'to' | 'from/to'
export type RawTrafficRule = {
allow: boolean
protocol: string
ipRange: string
direction: string
port?: string
}
type BaseTrafficRule = RawTrafficRule & {
id: string
networkId: XoNetwork['id']
}
export type TrafficRule =
| (BaseTrafficRule & { type: XoVif['type']; sourceId: XoVif['id'] })
| (BaseTrafficRule & { type: XoNetwork['type']; sourceId: XoNetwork['id'] })

View File

@@ -1,8 +1,5 @@
export * from './common.mjs'
export * from './xen-api.mjs'
export * from './xo.mjs'
export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/xen-orchestra-acl.mjs'
export * from './lib/complex-matcher.mjs'
export * from './lib/rest-api.mjs'

View File

@@ -1,102 +0,0 @@
import type { NextFunction, Request, Response } from 'express'
import type { VatesTask } from './vates-task.mjs'
import type { XoApp } from '../xo-app.mjs'
export type SecurityName = '*' | 'token' | 'basic' | 'none'
export type FieldDefinition =
| {
type: 'string'
example?: string
optional?: boolean
}
| {
type: 'boolean'
example?: boolean
optional?: boolean
}
| {
type: 'number'
example?: number
optional?: boolean
}
| {
type: 'enum'
enum: string[]
example?: string
optional?: boolean
}
| {
type: 'object'
fields: Record<string, FieldDefinition>
optional?: boolean
}
| {
type: 'array'
items: FieldDefinition
example?: unknown[]
optional?: boolean
}
export type ParamFieldDefinition = Exclude<
FieldDefinition,
{ type: 'boolean' } | { type: 'object' } | { type: 'array' }
>
export type QueryFieldDefinition = Exclude<FieldDefinition, { type: 'object' } | { type: 'array' }>
type RestApi = object & { xoApp: XoApp }
export type CreateAction = <Result>(
callback: (task: VatesTask) => MaybePromise<Result>,
opts: {
sync?: boolean
statusCode?: number
taskProperties: { name: string; [key: string]: unknown }
}
) => Promise<Result | undefined>
export interface BaseRouteDefinition<Middleware> {
method: 'get' | 'post' | 'put' | 'delete' | 'patch'
endpoint: string
description?: string
tags?: string[]
params?: Record<string, ParamFieldDefinition>
query?: Record<string, QueryFieldDefinition>
body?: Record<string, FieldDefinition>
responses?: Array<{
status: number
description: string
schema?: Record<string, FieldDefinition>
}>
middlewares?: Middleware[]
scope?: 'acl'
callback: (params: {
req: Request
res: Response
next: NextFunction
restApi: RestApi
createAction: CreateAction
}) => MaybePromise<unknown>
security?: SecurityName
}
export type MiddlewareDescriptor =
| { name: 'json' | 'urlencoded' | 'text' | 'raw'; options?: Record<string, unknown> }
| { name: 'acl'; acls: LooseAclEntry | LooseAclEntry[] }
type MaybePromise<T> = T | Promise<T>
/**
* Loosely-typed mirror of {@link https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts | AclEntry}
* from `@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts`.
*/
export type LooseAclEntry = {
resource: string
action?: string | ((opts: { req: object; restApi: RestApi }) => string | undefined)
actions?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
objectId?: string | ((opts: { req: object; restApi: RestApi }) => string)
objectIds?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
object?: object | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object> | undefined)
objects?: object[] | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object[]> | undefined)
getObject?: (opts: { restApi: RestApi }) => (id: string) => MaybePromise<object>
}
export type PluginRestRouteDefinition = BaseRouteDefinition<MiddlewareDescriptor>

View File

@@ -1,83 +0,0 @@
import { Branded } from '../common.mjs'
import type { XoGroup, XoUser } from '../xo.mjs'
type XoAclBaseRole = {
id: Branded<'acl-v2-role'>
name: string
description?: string
privilegeIds: XoAclBasePrivilege['id'][]
}
type XoAclRegularRole = XoAclBaseRole & {
groupIds: XoGroup['id'][]
userIds: XoUser['id'][]
}
type XoAclTemplateRole = XoAclBaseRole & {
isTemplate: true
roleTemplateId: number
}
export type XoAclRole = XoAclRegularRole | XoAclTemplateRole
export type XoAclSupportedActionsByResource = {
[resource: string]: Record<string, unknown>
}
/**
* E.g
* vms {
* shutdown: {
* clean: true,
* hard: true,
* }
* }
*
* `shutdown | shutdown:clean | shutdown:hard`
*/
export type GetKeysRecursively<T, Prefix extends string = ''> = {
[K in keyof T]: T[K] extends object
? K extends string
? `${Prefix}${K}` | GetKeysRecursively<T[K], `${Prefix}${K}:`>
: never
: K extends string
? `${Prefix}${K}`
: never
}[keyof T]
export type XoAclSupportedResource<TActionsByResource extends XoAclSupportedActionsByResource> =
keyof TActionsByResource
export type XoAclSupportedActions<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = (GetKeysRecursively<TActionsByResource[TResource]> & string) | '*'
export type XoAclBasePrivilege = {
id: Branded<'acl-v2-privilege'>
resource: string
action: string
selector?: string
effect: 'allow' | 'deny'
roleId: XoAclRole['id']
}
export type XoAclPrivilege<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = XoAclBasePrivilege & {
resource: TResource
action: XoAclSupportedActions<TActionsByResource, TResource>
}
export type XoUserRole = {
id: string
roleId: XoAclRole['id']
userId: XoUser['id']
}
export type XoGroupRole = {
id: string
roleId: XoAclRole['id']
groupId: XoGroup['id']
}

View File

@@ -1,103 +1,28 @@
import {
WrappedXenApiRecord,
XenApiHost,
XenApiHostWrapped,
XenApiNetwork,
XenApiNetworkWrapped,
XenApiRecord,
XenApiSr,
XenApiTask,
XenApiVbd,
XenApiVdi,
XenApiVif,
XenApiVm,
XenApiVmWrapped,
XenApiVtpm,
} from '../xen-api.mjs'
import type {
BOND_MODE,
OPAQUE_REF_NULL,
SUPPORTED_VDI_FORMAT,
VBD_MODE,
VBD_TYPE,
VDI_TYPE,
VIF_LOCKING_MODE,
VM_OPERATIONS,
} from '../common.mjs'
import type { PassThrough, Readable } from 'node:stream'
import type {
XapiXoRecord,
XoGpuGroup,
XoVgpuType,
XoHost,
XoNetwork,
XoPif,
XoSr,
XoUser,
XoVdi,
XoVgpuType,
XoVm,
XoVmTemplate,
XoVif,
XoVmSnapshot,
} from '../xo.mjs'
/**
* Properties accepted by {@link Xapi.editVm}.
*
* Field names match the canonical (camelCase) properties defined by the
* underlying `_editVm` (via `makeEditObject`). `_editVm` still resolves the
* snake_case and legacy aliases, but the canonical names are used here.
*/
export interface EditVmProps {
affinityHost?: string | null
autoPoweron?: boolean
blockedOperations?: Partial<Record<VM_OPERATIONS, boolean | string | null>>
coresPerSocket?: number | string | null
cpuCap?: number | null
cpuMask?: number[]
cpuWeight?: number | null
cpus?: number
cpusStaticMax?: number | string
/**
* Update VM creation metadata stored under `other_config.xo:*`. The object is
* merged with the existing data.
*/
creation?: { user?: string }
expNestedHvm?: boolean
hasVendorDevice?: boolean
highAvailability?: 'best-effort' | 'restart' | ''
hvmBootFirmware?: string | null
memory?: number | string
memoryMax?: number | string
memoryMin?: number | string
memoryStaticMax?: number | string
nameDescription?: string
nameLabel?: string
nestedVirt?: boolean
nicType?: string | null
notes?: string | null
PV_args?: string
secureBoot?: boolean
startDelay?: number
suspendSr?: string | null
tags?: string[]
uefiMode?: 'setup' | 'user'
vga?: 'std' | 'cirrus'
videoram?: 1 | 2 | 4 | 8 | 16
viridian?: boolean
virtualizationMode?: 'pv' | 'hvm'
xenStoreData?: Record<string, string | null>
}
/** Properties accepted by {@link Xapi.editVif}. */
export interface EditVifProps {
ipv4Allowed?: string[]
ipv6Allowed?: string[]
lockingMode?: VIF_LOCKING_MODE
// in kB/s, `null` removes the limit
rateLimit?: number | null
txChecksumming?: boolean
}
import type { SUPPORTED_VDI_FORMAT } from '../common.mjs'
export type XcpPatches = {
changelog?: {
@@ -129,29 +54,14 @@ export type XsPatches = {
}
export interface Xapi {
status: string
pool?: { uuid: string }
sessionId: string
_auth: { user: string; password: string }
_url: { protocol: string; hostname: string; hostnameRaw: string; port?: string }
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
connectVif(vifId: XoVif['id']): Promise<void>
getField<T extends XenApiRecord, K extends keyof T>(
type: Extract<WrappedXenApiRecord, T>['$type'],
ref: T['$ref'],
field: K
): Promise<T[K]>
getObject: <
XoRecord extends XapiXoRecord,
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
>(
idOrUuidOrRef: XoRecord['id'] | WrappedRecord['$ref'] | WrappedRecord['uuid'],
defaultValue?: WrappedRecord
) => WrappedRecord
createNetwork(
params:
| {
@@ -168,101 +78,12 @@ export interface Xapi {
vlan: number
}
): Promise<XenApiNetworkWrapped>
createBondedNetwork(params: {
bondMode: BOND_MODE
pifIds: XoPif['id'][]
name: string
description?: string
mtu?: number
}): Promise<XenApiNetworkWrapped>
deleteNetwork(id: XoNetwork['id']): Promise<void>
deleteVif(vifId: XoVif['id']): Promise<void>
disconnectVif(vifId: XoVif['id']): Promise<void>
exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
emergencyShutdownHost(hostId: XoHost['id']): Promise<void>
ejectHostFromPool(hostId: XoHost['id']): Promise<void>
host_smartReboot(
hostRef: XenApiHost['$ref'],
bypassBlockedSuspend?: boolean,
bypassCurrentVmCheck?: boolean
): Promise<void>
migrateVm(
vmId: XoVm['id'],
hostXapi: Xapi,
hostId: XoHost['id'],
opts?: {
force?: boolean
mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
migrationNetworkId?: XoNetwork['id']
sr?: XoSr['id']
bypassAssert?: boolean
}
): Promise<void>
joinPool(masterAddress: string, masterUsername: string, masterPassword: string, force?: boolean): Promise<void>
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
pool_emergencyShutdown(): Promise<void>
resumeVm(id: XoVm['id']): Promise<void>
revertVm(snapshotId: XoVmSnapshot['id']): Promise<void>
unpauseVm(id: XoVm['id']): Promise<void>
cloneVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
fast?: boolean
}
): Promise<XenApiVmWrapped>
copyVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
srOrSrId?: XoSr['id'] | XoSr
}
): Promise<XenApiVmWrapped>
remoteCopyVm(
vmId: XoVm['id'],
targetXapi: Xapi,
targetSrId: XoSr['id'],
opts?: {
compress?: 'zstd' | 'gzip' | boolean
nameLabel?: string
}
): Promise<{ vm: XenApiVmWrapped }>
forgetSr(id: XoSr['id']): Promise<void>
forgetHost(hostId: XoHost['id']): Promise<void>
powerOnHost(hostId: XoHost['id']): Promise<void>
rebootHost(hostId: XoHost['id'], force?: boolean): Promise<void>
shutdownHost(hostId: XoHost['id'], opts?: { force?: boolean; bypassEvacuate?: boolean }): Promise<void>
SR_create(params: {
content_type?: XoSr['content_type']
// index signature, not `Record<string, string>`: tsoa emits an empty schema for `Record`
// (https://github.com/lukeautry/tsoa/pull/1858). device_config keys are driver-specific (SR_type).
device_config: {
chapuser?: string
chappassword?: string
device?: string
legacy_mode?: string
location?: string
nfsversion?: string
options?: string
password?: string
SCSIid?: string
server?: string
serverpath?: string
target?: string
targetIQN?: string
username?: string
} & {
[key: string]: string
}
host: XenApiHost['$ref']
name_description?: XoSr['name_description']
name_label: XoSr['name_label']
physical_size?: number
shared: boolean
sm_config?: { [key: string]: string }
type: XoSr['SR_type']
}): Promise<XenApiSr['$ref']>
SR_importVdi(
ref: XenApiSr['$ref'],
stream: Readable,
@@ -280,13 +101,6 @@ export interface Xapi {
xenstore_data?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
editVm(
id: XoVm['id'],
props: EditVmProps,
checkLimits?: (limits: Record<string, number>, vm: XenApiVmWrapped) => Promise<void>
): Promise<void>
editVif(id: XoVif['id'], props: EditVifProps): Promise<void>
startVm(
id: XoVm['id'],
opts?: {
@@ -304,7 +118,7 @@ export interface Xapi {
): Promise<void>
VM_export(
vmRef: XenApiVm['$ref'],
opts?: { cancelToken?: unknown; compress?: boolean | 'zstd' | 'gzip'; useSnapshot?: boolean }
opts?: { cancelToken?: unknown; compress?: boolean; useSnapshot?: boolean }
): ReturnType<Xapi['getResource']>
VM_import(
stream: Readable,
@@ -316,7 +130,6 @@ export interface Xapi {
metadataVm: {
affinityHost?: XoHost['id']
autoPoweron?: boolean
cpus?: number
memory?: number
name_description?: string
name_label: string
@@ -358,51 +171,11 @@ export interface Xapi {
vgpuType?: XoVgpuType['id']
gpuGroup?: XoGpuGroup['id']
copyHostBiosStrings?: boolean
highAvailability?: XoVm['high_availability']
hvmBootFirmware?: 'uefi' | 'bios'
secureBoot?: boolean
},
checkLimits?: boolean,
creatorId?: XoUser['id'],
opts?: { destroyAllVifs: boolean }
): Promise<XenApiVmWrapped>
moveVdi(
vdiId: XenApiVdi['$ref'] | XoVdi['id'],
srId: XenApiSr['$ref'] | XoSr['id'],
opts?: { _failOnCbtError?: boolean }
): Promise<XenApiVdi>
VBD_create(params: {
bootable?: boolean
empty?: boolean
mode?: VBD_MODE
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
type?: VBD_TYPE
unpluggable?: boolean
userdevice?: string
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
VM: XenApiVm['$ref']
}): Promise<XenApiVbd['$ref']>
VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
VDI_create(
options: {
name_description?: XoVdi['name_description']
name_label?: XoVdi['name_label']
other_config?: XoVdi['other_config']
read_only?: boolean
sharable?: boolean
SR?: XenApiSr['$ref']
tags?: XoVdi['tags']
type?: VDI_TYPE
virtual_size: XoVdi['size']
xenstore_data?: Record<string, string>
},
extraOptions?: {
sm_config?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
VDI_exportContent(
@@ -411,27 +184,9 @@ export interface Xapi {
): Promise<Readable & { length?: number }>
VDI_importContent(
vdiRef: XenApiVdi['$ref'],
stream: Readable | Buffer,
stream: Readable,
opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
): Promise<void>
VIF_create(
options: {
currently_attached?: boolean
device?: string
ipv4_allowed?: string[]
ipv6_allowed?: string[]
locking_mode?: VIF_LOCKING_MODE
MTU?: number
network: XenApiNetwork['$ref']
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
VM: XenApiVm['$ref']
},
extraOptions?: {
MAC?: string
}
): Promise<XenApiVif['$ref']>
VM_createCloudInitConfig(
vmRef: XenApiVm['$ref'],
cloudConfig: string,
@@ -450,29 +205,5 @@ export interface Xapi {
pathname: string,
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
): Promise<{ body: Readable }>
clearHost(
host: Pick<XenApiHostWrapped, '$ref' | '$pool'>,
force?: boolean,
opts?: { transient?: boolean }
): Promise<void>
disableHost(hostId: XoHost['id'], opts?: { transient?: boolean }): Promise<void>
enableHost(hostId: XoHost['id']): Promise<void>
getRecordByUuid<
Type extends WrappedXenApiRecord['$type'],
XenApiRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: Type }>,
>(
type: Type,
uuid: XenApiRecord['uuid']
): Promise<XenApiRecord>
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
destroySr(id: XoSr['id']): Promise<void>
xostor_delete(ref: XenApiSr['$ref']): Promise<void>
objects: {
indexes: {
type: {
[XenApiRecord in WrappedXenApiRecord as XenApiRecord['$type']]: Record<XenApiRecord['uuid'], XenApiRecord>
}
}
}
}

View File

@@ -82,7 +82,6 @@ type TagCallMethods = {
type WrapperXenApi<T, Type extends string, Fn = { (): void }> = T & {
$call: Fn
$callAsync: Fn
$pool: XenApiPool
$type: Type
$snapshot(params: {
cancelToken?: unknown
@@ -192,7 +191,6 @@ export interface XenApiPool {
ha_host_failures_to_tolerate: number
ha_overcommitted: boolean
ha_plan_exists_for: number
ha_reboot_vm_on_internal_shutdown?: boolean
ha_statefiles: string[]
health_check_config: Record<string, string>
igmp_snooping_enabled?: boolean
@@ -492,9 +490,7 @@ export interface XenApiDrTask {
uuid: string
}
type XenApiHostCallMethods = TagCallMethods & {
<T>(method: 'call_plugin', plugin: string, fn: string, args: Record<string, string>): Promise<T>
}
type XenApiHostCallMethods = TagCallMethods & {}
export interface XenApiHost {
$ref: Branded<'host'>
address: string
@@ -571,9 +567,7 @@ export interface XenApiHost {
uuid: string
virtual_hardware_platform_versions: number[]
}
export interface XenApiHostWrapped extends WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods> {
$restartAgent(): Promise<void>
}
export type XenApiHostWrapped = WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods>
export interface XenApiHostCrashdump {
$ref: Branded<'host_crashdump'>
@@ -628,9 +622,7 @@ export interface XenApiHostCpu {
vendor: string
}
type XenApiNetworkCallMethods = TagCallMethods & {
(method: 'add_purpose', purpose: NETWORK_PURPOSE): Promise<void>
}
type XenApiNetworkCallMethods = TagCallMethods & {}
export interface XenApiNetwork {
$ref: Branded<'network'>
@@ -1016,7 +1008,6 @@ export interface XenApiMessage {
timestamp: string
uuid: string
}
export type XenApiMessageWrapped = WrapperXenApi<XenApiMessage, 'message'>
export interface XenApiSecret {
$ref: Branded<'secret'>

View File

@@ -1,416 +0,0 @@
import { EventEmitter } from 'node:stream'
import type {
AnyXoJob,
AnyXoLog,
XapiXoRecord,
XoAuthenticationToken,
XoBackupRepository,
XoConfigBackupArchive,
XoGroup,
XoHost,
XoPool,
XoPoolBackupArchive,
XoProxy,
XoSchedule,
XoServer,
XoTask,
XoUser,
XoVif,
XoVm,
XoVmBackupArchive,
} from './xo.mjs'
import { VatesTask } from './lib/vates-task.mjs'
import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
import {
Xapi,
XapiHostStats,
XapiPoolStats,
XapiStatsGranularity,
XapiVmStats,
XenApiGpuGroupWrapped,
XenApiHostWrapped,
XenApiMessageWrapped,
XenApiNetworkWrapped,
XenApiPbdWrapped,
XenApiPciWrapped,
XenApiPgpuWrapped,
XenApiPifWrapped,
XenApiPoolWrapped,
XenApiSmWrapped,
XenApiSrWrapped,
XenApiVbdWrapped,
XenApiVdiWrapped,
XenApiVgpuTypeWrapped,
XenApiVgpuWrapped,
XenApiVifWrapped,
XenApiVmWrapped,
XenApiVtpmWrapped,
XoAclBasePrivilege,
XoAclRole,
XoGroupRole,
XoUserRole,
} from './index.mjs'
type FeatureCode =
| 'BACKUP.DELTA'
| 'BACKUP.DELTA_REPLICATION'
| 'BACKUP.FULL'
| 'BACKUP.HEALTHCHECK'
| 'BACKUP.METADATA'
| 'BACKUP.MIRROR'
| 'BACKUP.WITH_RAM'
| 'BACKUP.SMART_BACKUP'
| 'BACKUP.S3'
| 'DOCKER'
| 'EXPORT.XVA'
| 'LIST_MISSING_PATCHES'
| 'POOL_EMERGENCY_SHUTDOWN'
| 'RBAC'
| 'ROLLING_POOL_UPDATE'
| 'ROLLING_POOL_REBOOT'
| 'SMART_REBOOT'
| 'WARM_MIGRATION'
| 'PLUGIN.OPENMETRICS'
type XapiRecordByXapiXoRecord = {
gpuGroup: XenApiGpuGroupWrapped
host: XenApiHostWrapped
message: XenApiMessageWrapped
network: XenApiNetworkWrapped
PBD: XenApiPbdWrapped
PCI: XenApiPciWrapped
PGPU: XenApiPgpuWrapped
PIF: XenApiPifWrapped
pool: XenApiPoolWrapped
SR: XenApiSrWrapped
SM: XenApiSmWrapped
VBD: XenApiVbdWrapped
VDI: XenApiVdiWrapped
'VDI-snapshot': XenApiVdiWrapped
'VDI-unmanaged': XenApiVdiWrapped
vgpu: XenApiVgpuWrapped
vgpuType: XenApiVgpuTypeWrapped
VIF: XenApiVifWrapped
VM: XenApiVmWrapped
'VM-controller': XenApiVmWrapped
'VM-snapshot': XenApiVmWrapped
'VM-template': XenApiVmWrapped
VTPM: XenApiVtpmWrapped
}
type LicenseProductId =
| 'premium'
| 'xcpng-enterprise'
| 'xcpng-standard'
| 'xo-proxy'
| 'xosan.trial'
| 'xostor'
| 'xostor.trial'
type LicenseProductType = 'xo' | 'xoproxy' | 'xcpng' | 'xosan' | 'xostor'
type License = {
id: string
type: 'license'
created: number
licenseType: 'unit'
productId: LicenseProductId
tags: string[]
expires?: number
boundObjectId?: string
bindDate?: number
buyer?: { token: string; email: string }
history?: { date: number; boundObjectId: string }[]
rebound?: number
productTypes?: LicenseProductType[]
bundleInfo?: { name: string; id: string }
}
export type XoApp = {
hooks: EventEmitter
_redis: {
get(key: string): Promise<string | null>
mSet(args: string[]): Promise<unknown>
sMembers(key: string): Promise<string[]>
sIsMember(key: string, member: string): Promise<boolean>
}
config: {
get<T = string>(path: string): T
getOptional<T = unknown>(path: string): T | undefined
getOptionalDuration(path: string): number | undefined
getGuiRoutes(): Promise<{
default: {
url: string
path: string
}
v5?: {
url: string
path: string
}
v6?: {
url: string
path: string
}
[key: string]:
| undefined
| {
url: string
path: string
}
}>
}
objects: EventEmitter & {
allIndexes: {
type: {
getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
}
}
}
tasks: EventEmitter & {
abort(id: XoTask['id'], reason?: string): Promise<void>
clearLogs(): Promise<void>
create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
deleteLog(id: XoTask['id']): Promise<void>
get(id: XoTask['id']): Promise<XoTask>
list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
}
apiContext: {
user?: XoUser
permission?: XoUser['permission'] | 'none' | null
}
// methods ------------
allocIpAddresses(vifId: XoVif['id'], addAddresses?: string[], removeAddresses?: string[]): Promise<void>
addAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<XoGroupRole>
addAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<XoUserRole>
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
addApiMethod: <A extends unknown[], R>(
name: string,
method: (...args: A) => Promise<R>,
//we use any since it is a legacy call, might be better typed in another pr
info: {
resolve?: any
params?: any
}
) => () => void // eslint-disable-line @typescript-eslint/no-explicit-any
registerRestRoutes: (routes: PluginRestRouteDefinition[], base?: string) => () => void
authenticateUser: (
credentials: { token?: string; username?: string; password?: string },
userData?: { ip?: string },
opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
backupGuard(poolId: XoPool['id']): Promise<void>
/* Throw if no authorization */
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
/* connect a server (XCP-ng/XenServer) */
connectXenServer(id: XoServer['id']): Promise<void>
// TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)
createAclV2Privilege(
privilege: Omit<XoAclBasePrivilege, 'id'>,
options?: {
force?: boolean
}
): Promise<XoAclBasePrivilege>
copyAclV2Role(
id: XoAclRole['id'],
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
): Promise<XoAclRole['id']>
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
createAuthenticationToken(opts: {
client?: {
id?: string
[key: string]: unknown
}
description?: string
expiresIn?: string | number
userId: XoUser['id']
}): Promise<XoAuthenticationToken>
createRemote(params: {
name: string
options?: string
proxy?: XoProxy['id']
url: string
}): Promise<XoBackupRepository>
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
deleteAclV2GroupRole(
groupId: XoGroup['id'],
roleId: XoAclRole['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<boolean>
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
detachHostFromPool(hostId: XoHost['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
/* disconnect a server (XCP-ng/XenServer) */
disconnectXenServer(id: XoServer['id']): Promise<void>
findEnabledScheduleSequenceFromSchedule(id: XoSchedule['id']): Promise<XoSchedule | undefined>
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
getAclV2RolePrivileges(roleId: XoAclRole['id']): Promise<XoAclBasePrivilege[]>
getAclV2Role(id: XoAclRole['id'], opts?: { bypassAuthorization?: boolean }): Promise<XoAclRole>
deleteAclV2UserRole(
userId: XoUser['id'],
roleId: XoAclRole['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<boolean>
getAclV2GroupRoles(
groupId: XoGroup['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<Exclude<XoAclRole, { isTemplate: true }>[]>
getAclV2Roles(): Promise<XoAclRole[]>
getAclV2UserPrivileges(userId: XoUser['id']): Promise<XoAclBasePrivilege[]>
getAclV2UserRoles(
userId: XoUser['id'],
opts?: { bypassAuthorization?: boolean; fromGroup?: boolean; fromUser?: boolean }
): Promise<XoAclRole[]>
getAllGroups(): Promise<XoGroup[]>
getAllProxies(): Promise<XoProxy[]>
getAllJobs<T extends AnyXoJob['type']>(type: T): Promise<Extract<AnyXoJob, { type: T }>[]>
getAllJobs(type?: string): Promise<AnyXoJob[]>
getProxy(id: XoProxy['id']): Promise<XoProxy>
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
getAllRemotes(): Promise<XoBackupRepository[]>
removeRemote(id: XoBackupRepository['id']): Promise<void>
getAllRemotesInfo(): Promise<
Record<
XoBackupRepository['id'],
{
size?: number
used?: number
available?: number
encryption?: {
algorithm: string
isLegacy: boolean
recommanded: string
}
}
>
>
getAllSchedules(): Promise<XoSchedule[]>
getAllUsers(): Promise<XoUser[]>
getAllXenServers(): Promise<XoServer[]>
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
getBackupNgLogsSorted(opts: {
after?: number
before?: number
filter: (log: AnyXoLog) => boolean
limit?: number
}): Promise<AnyXoLog[]>
getGroup(id: XoGroup['id']): Promise<XoGroup>
getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
getObjectsByType: <T extends XapiXoRecord>(
type: T['type'],
opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
) => Record<T['id'], T> | undefined
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
getUser: (id: XoUser['id']) => Promise<XoUser>
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
getXapiObject: <T extends XapiXoRecord>(
maybeId: T['id'] | T,
type: T['type'] | T['type'][]
) => XapiRecordByXapiXoRecord[T['type']]
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
getXenServer(id: XoServer['id']): Promise<XoServer>
hasFeatureAuthorization(featureCode: string): Promise<boolean>
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
}>
listVmBackupsNg(
backupRepositoryIds: XoBackupRepository['id'][],
opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
/** Allow to add a new server in the DB (XCP-ng/XenServer) */
registerXenServer(
body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
allowUnauthorized?: XoServer['allowUnauthorized']
password: string
readOnly?: XoServer['readOnly']
}
): Promise<XoServer>
rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask; shutdownPinnedVms?: boolean }): Promise<void>
rollingPoolUpdate(
pool: XoPool,
opts?: { rebootVm?: boolean; parentTask?: VatesTask; shutdownPinnedVms?: boolean }
): Promise<void>
setVmResourceSet(vmId: XoVm['id'], resourceSetId: string | null, force?: boolean): Promise<void>
shareVmResourceSet(vmId: XoVm['id']): Promise<void>
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
runJob(job: AnyXoJob, schedule: XoSchedule): void
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
testRemote(
id: XoBackupRepository['id']
): Promise<
| { success: true; readRate: number; writeRate: number }
| { success: false; step: string; file: string; error: unknown }
>
/** Remove a server from the DB (XCP-ng/XenServer) */
unregisterXenServer(id: XoServer['id']): Promise<void>
updateUser(
id: XoUser['id'],
updates: {
/**
* @deprecated
*/
email?: string
authProviders?: Record<string, string>
name?: string
password?: string
permission?: string
preferences?: Record<string, string>
}
): Promise<void>
updateAclV2Privilege(
privilegeId: XoAclBasePrivilege['id'],
privilege: XoAclBasePrivilege
): Promise<XoAclBasePrivilege>
updateAclV2Role(
roleId: XoAclRole['id'],
role: {
name?: XoAclRole['name']
description?: XoAclRole['description'] | null
},
options?: {
force?: boolean
}
): Promise<XoAclRole>
updateGroup(
id: XoGroup['id'],
updates: {
name?: string
}
): void
updateRemote(
id: XoBackupRepository['id'],
params: {
enabled?: boolean
name?: string
options?: string | null
proxy?: XoProxy['id'] | null
url?: string
}
): Promise<XoBackupRepository>
getAllXapis(): Record<string, Xapi>
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
getLicenses(params?: { productType?: LicenseProductType }): Promise<License[]>
bindLicense(params: { licenseId: string; boundObjectId: string }): Promise<License>
unbindLicense(params: {
productId: LicenseProductId
boundObjectId: string
licenseId: string
data?: Record<string, string>
}): Promise<License>
}

View File

@@ -19,7 +19,6 @@ import type {
VM_POWER_STATE,
} from './common.mjs'
import type * as CMType from './lib/complex-matcher.mjs'
import { XoAclPrivilege, XoAclRole, XoAclSupportedActionsByResource } from './lib/xen-orchestra-acl.mjs'
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
type BaseXapiXo = {
@@ -76,7 +75,7 @@ type BaseXoVm = BaseXapiXo & {
expNestedHvm: boolean
isNestedVirtEnabled: boolean
hasVendorDevice: boolean
high_availability: 'best-effort' | 'restart' | ''
high_availability: string
installTime?: number | null
isFirmwareSupported: boolean
memory: {
@@ -94,7 +93,7 @@ type BaseXoVm = BaseXapiXo & {
notes?: string
os_version: Record<string, string> | null
other: Record<string, string>
parent?: XoVmSnapshot['id']
parent?: XoVm['id']
power_state: VM_POWER_STATE
pvDriversDetected?: boolean
pvDriversUpToDate?: boolean
@@ -107,10 +106,6 @@ type BaseXoVm = BaseXapiXo & {
suspendSr?: XoSr['id']
tags: string[]
vga?: string
/**
* Value in MiB.
* See: https://wiki.xenproject.org/wiki/XCP_PV_templates_start
*/
videoram?: number
viridian: boolean
virtualizationMode: DOMAIN_TYPE
@@ -142,16 +137,10 @@ export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
// TODO: to be typed when Bastien.N has finished working on the XO task
type BaseXoLog = {
id: Branded<'xo-log'>
infos?: { data: unknown; message: string }[]
[key: string]: unknown
}
export type XoBackupLog = BaseXoLog & {
status: 'success' | 'skipped' | 'interrupted' | 'failure' | 'pending'
message: 'backup'
start: number
end?: number
tasks?: XoTask[]
jobId?: AnyXoBackupJob['id']
message: 'backup' | 'metadata'
}
export type XoRestoreLog = BaseXoLog & {
message: 'restore'
@@ -171,7 +160,6 @@ export type XoVmBackupArchive = {
uuid: XoVm['uuid']
name_description: string
name_label: string
tags: XoVm['tags']
}
differencingVhds?: number
dynamicVhds?: number
@@ -222,7 +210,6 @@ export type XoGroup = {
provider?: string
providerGroupId?: string
users: XoUser['id'][]
aclRoleIds: XoAclRole['id'][]
}
export type XoHost = BaseXapiXo & {
@@ -241,7 +228,7 @@ export type XoHost = BaseXapiXo & {
address: string
agentStartTime: null | number
bios_strings: Record<string, string>
bios_string: Record<string, string>
build: string
certificates?: {
fingerprint: string
@@ -250,7 +237,7 @@ export type XoHost = BaseXapiXo & {
chipset_info: {
iommu?: boolean
}
controlDomain?: XoVmController['id']
controlDomain?: XoVm['id']
cpus: {
cores?: number
sockets?: number
@@ -336,7 +323,6 @@ export type XoNetwork = BaseXapiXo & {
defaultIsLocked: boolean
id: Branded<'network'>
insecureNbd?: boolean
isBonded: boolean
name_description: string
name_label: string
nbd?: boolean
@@ -399,7 +385,7 @@ export type XoPif = BaseXapiXo & {
attached: boolean
bondMaster?: XoPif['id']
bondSlaves?: XoPif['id'][]
bondSalves?: XoPif['id'][]
carrier: boolean
device: string
deviceName?: string
@@ -432,9 +418,8 @@ export type XoPool = BaseXapiXo & {
}
crashDumpSr?: XoSr['id']
current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
default_SR?: XoSr['id']
defaultSr?: XoSr['id']
HA_enabled: boolean
haRebootVmOnInternalShutdown: boolean
haSrs: XoSr['id'][]
id: Branded<'pool'>
master: XoHost['id']
@@ -447,6 +432,7 @@ export type XoPool = BaseXapiXo & {
tags: string[]
type: 'pool'
vtpmSupported: boolean
xosanPackInstallationTime: number | null
zstdSupported: boolean
}
@@ -469,7 +455,6 @@ type BaseXoJob = {
type XoBackupJobGeneralSettings = {
backupReportTpl?: 'compactMjml' | 'mjml'
reportWhen?: 'always' | 'error' | 'failure' | 'never'
reportRecipients?: string[]
hideSuccessfulItems?: boolean
[key: string]: unknown
}
@@ -483,8 +468,6 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
checkpointSnapshot?: boolean
offlineSnapshot?: boolean
maxExportRate?: number
nbdConcurrency?: number
nRetriesVmBackupFailures?: number
@@ -493,6 +476,7 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
mergeBackupsSynchronously?: boolean
offlineBackup?: boolean
timeout?: number
[key: string]: unknown
}
export type XoVmBackupJobScheduleSettings = {
exportRetention?: number
@@ -540,11 +524,11 @@ export type XoMetadataBackupJob = BaseXoJob & {
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
concurrency?: number
nRetriesVmBackupFailures?: number
mergeBackupsSynchronously?: boolean
timeout?: number
maxExportRate?: number
backupReportTpl?: 'compactMjml'
reportWhen: 'failure'
[key: string]: unknown
}
export type XoMirrorBackupScheduleSettings = {
exportRetention?: number
@@ -557,28 +541,13 @@ export type XoMirrorBackupJob = BaseXoJob & {
mode: 'full' | 'delta'
sourceRemote: XoBackupRepository['id']
remotes: CMType.IdOr<XoBackupRepository['id']>
proxy?: XoProxy['id']
settings: {
'': XoMirrorBackupGeneralSettings
[scheduleId: XoSchedule['id']]: XoMirrorBackupScheduleSettings | undefined
}
}
export type XoJob = BaseXoJob & {
type: 'call'
method: string
paramsVector?: {
type: string
items: {
type?: string
values?: {
schedules?: XoSchedule['id'][]
[key: string]: unknown
}[]
[key: string]: unknown
}[]
}
}
export type XoJob = BaseXoJob & {}
export type XoSchedule = {
cron: string
@@ -614,7 +583,7 @@ export type XoSr = BaseXapiXo & {
VDIs: AnyXoVdi['id'][]
allocationStrategy?: 'thin' | 'thick' | 'unknown'
content_type: 'user' | 'iso' | 'disk'
content_type: string
current_operations: Record<string, STORAGE_OPERATIONS>
id: Branded<'SR'>
inMaintenanceMode: boolean
@@ -625,28 +594,7 @@ export type XoSr = BaseXapiXo & {
shared: boolean
size: number
sm_config: Record<string, string>
// SM drivers shipped with XCP-ng/XenServer
SR_type:
| 'cephfs'
| 'dummy'
| 'ext'
| 'file'
| 'glusterfs'
| 'hba'
| 'iso'
| 'largeblock'
| 'linstor'
| 'lvm'
| 'lvmofcoe'
| 'lvmohba'
| 'lvmoiscsi'
| 'moosefs'
| 'nfs'
| 'shm'
| 'smb'
| 'udev'
| 'xfs'
| 'zfs'
SR_type: string
tags: string[]
type: 'SR'
usage: number
@@ -669,39 +617,19 @@ export type XoSm = BaseXapiXo & {
export type XoTask = {
abortionRequestedAt?: number
data?: Record<string, string>
end?: number
id: Branded<'task'>
infos?: { data: unknown; message: string }[]
progress?: number
properties: {
method?: string
name?: string
objectId?: string
objectType?:
| XapiXoRecord['type']
| 'backup'
| 'backup-archive'
| 'backup-job'
| 'backup-log'
| 'backup-repository'
| 'group'
| 'proxy'
| 'restore'
| 'restore-log'
| 'schedule'
| 'server'
| 'task'
| 'user'
| 'acl-privilege'
| 'acl-role'
params?: Record<string, unknown>
progress?: number
type?: string
userId?: string
[key: string]: unknown | undefined
}
result?: Record<string, unknown>
result: Record<string, unknown>
start: number
status: 'failure' | 'interrupted' | 'pending' | 'success'
tasks?: XoTask[]
@@ -715,7 +643,7 @@ export type XoUser = {
groups: XoGroup['id'][]
id: Branded<'user'>
name?: string
permission: 'none' | 'admin'
permission: string
pw_hash?: string
preferences: Record<string, string>
}
@@ -768,7 +696,6 @@ type BaseXoVdi = BaseXapiXo & {
export type XoVdi = BaseXoVdi & {
id: Branded<'VDI'>
chainPhysicalUsage: number
type: 'VDI'
}
@@ -889,28 +816,22 @@ export type XapiXoRecord =
| XoVtpm
| XoSm
export type NonXapiXoRecord<
TActionsByResource extends XoAclSupportedActionsByResource = never,
TResource extends string = never,
> =
export type NonXapiXoRecord =
| AnyXoBackupArchive
| AnyXoJob
| AnyXoLog
| XoGroup
| XoProxy
| XoGroup
| XoProxy
| XoJob
| XoBackupRepository
| XoSchedule
| XoServer
| XoTask
| XoUser
| XoAclRole
| XoAclPrivilege<TActionsByResource, TResource>
export type XoRecord<
TActionsByResource extends XoAclSupportedActionsByResource = never,
TResource extends string = never,
> = XapiXoRecord | NonXapiXoRecord<TActionsByResource, TResource>
export type XoRecord = XapiXoRecord | NonXapiXoRecord
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,16 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
}
})

View File

@@ -1,47 +0,0 @@
{
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"main": "dist/index.mjs",
"name": "@xen-orchestra/acl",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
"version": "1.5.0",
"license": "AGPL-3.0-or-later",
"private": false,
"type": "module",
"engines": {
"node": ">= 20.18"
},
"exports": {
".": {
"types": "./dist/index.d.mts",
"default": "./dist/index.mjs"
}
},
"scripts": {
"build": "tsc && node scripts/generate-types.mjs",
"clean": "rimraf dist/",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "npm run build && node --test ./dist/**/*.test.mjs",
"prebuild": "npm run clean",
"prepublishOnly": "npm run build"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"@vates/types": "^1.30.1",
"rimraf": "^6.0.1",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0"
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/acl",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"dependencies": {
"complex-matcher": "^1.0.0"
}
}

View File

@@ -1,55 +0,0 @@
#!/usr/bin/env node
// AUTO-GENERATES dist/generated/privilege-types.d.mts
// Run via `npm run build` (after tsc).
import { mkdirSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../dist/actions/index.mjs'
import { getActionStrings } from '../dist/index.mjs'
function toPascalCase(str) {
return str
.split(/[-_]/)
.map(s => s.charAt(0).toUpperCase() + s.slice(1))
.join('')
}
const resources = Object.keys(SUPPORTED_ACTIONS_BY_RESOURCE)
const typeBlocks = resources.map(resource => {
const actionUnion = getActionStrings(resource)
.map(action => `'${action}'`)
.join(' | ')
return `export type ${toPascalCase(resource)}Privilege = {
id: XoAclBasePrivilege['id']
resource: '${resource}'
action: ${actionUnion}
selector?: XoAclBasePrivilege['selector']
effect: XoAclBasePrivilege['effect']
roleId: XoAclBasePrivilege['roleId']
}`
})
const anyUnion = resources.map(resource => ` | ${toPascalCase(resource)}Privilege`).join('\n')
const types = `\
// -----------------------------------------------------------------------
// AUTO-GENERATED — do not edit manually.
// Regenerated on every \`npm run build\`.
// -----------------------------------------------------------------------
import type { XoAclBasePrivilege } from '@vates/types'
${typeBlocks.join('\n\n')}
export type AnyPrivilege =
${anyUnion}
`
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
const outDir = join(root, 'dist/generated')
mkdirSync(outDir, { recursive: true })
writeFileSync(join(outDir, 'privilege-types.d.mts'), types, 'utf-8')

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
action: true,
effect: true,
resource: true,
selector: true,
},
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
description: true,
groups: true,
name: true,
users: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,13 +0,0 @@
export default {
benchmark: true,
create: true,
read: true,
forget: true,
update: {
enabled: true,
name: true,
options: true,
proxy: true,
url: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,9 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
users: true,
},
}

View File

@@ -1,28 +0,0 @@
export default {
'allow-vm': true,
detach: true,
disable: true,
enable: true,
evacuate: true,
export: {
logs: true,
},
forget: true,
'join-pool': true,
'migrate-receive': true,
read: true,
reboot: {
clean: true,
smart: true,
},
'restart-toolstack': true,
'scan-pifs': true,
shutdown: {
clean: true,
emergency: true,
},
start: true,
update: {
tags: true,
},
}

View File

@@ -1,80 +0,0 @@
import { default as aclPrivilege } from './acl-privilege.mjs'
import { default as aclRole } from './acl-role.mjs'
import { default as alarm } from './alarm.mjs'
import { default as backupArchive } from './backup-archive.mjs'
import { default as backupJob } from './backup-job.mjs'
import { default as backupLog } from './backup-log.mjs'
import { default as backupRepository } from './backup-repository.mjs'
import { default as gpuGroup } from './gpuGroup.mjs'
import { default as group } from './group.mjs'
import { default as host } from './host.mjs'
import { default as message } from './message.mjs'
import { default as network } from './network.mjs'
import { default as pbd } from './pbd.mjs'
import { default as pci } from './pci.mjs'
import { default as pgpu } from './pgpu.mjs'
import { default as pif } from './pif.mjs'
import { default as pool } from './pool.mjs'
import { default as proxy } from './proxy.mjs'
import { default as restoreLog } from './restore-log.mjs'
import { default as schedule } from './schedule.mjs'
import { default as server } from './server.mjs'
import { default as sm } from './sm.mjs'
import { default as sr } from './sr.mjs'
import { default as task } from './task.mjs'
import { default as user } from './user.mjs'
import { default as vbd } from './vbd.mjs'
import { default as vdiSnapshot } from './vdi-snapshot.mjs'
import { default as vdiUnmanaged } from './vdi-unmanaged.mjs'
import { default as vdi } from './vdi.mjs'
import { default as vgpu } from './vgpu.mjs'
import { default as vgpuType } from './vgpuType.mjs'
import { default as vif } from './vif.mjs'
import { default as vmController } from './vm-controller.mjs'
import { default as vmSnapshot } from './vm-snapshot.mjs'
import { default as vmTemplate } from './vm-template.mjs'
import { default as vm } from './vm.mjs'
import { default as vtpm } from './vtpm.mjs'
/**
* value without sub action like 'shutdown' mean: 'shutdown:*' (so shutdown mean you can shutdown:clean and shutdown:hard)
*/
export const SUPPORTED_ACTIONS_BY_RESOURCE = {
'acl-privilege': aclPrivilege,
'acl-role': aclRole,
alarm,
'backup-archive': backupArchive,
'backup-job': backupJob,
'backup-log': backupLog,
'backup-repository': backupRepository,
group,
gpuGroup,
host,
message,
network,
pbd,
pci,
pgpu,
pif,
pool,
proxy,
'restore-log': restoreLog,
schedule,
server,
sm,
sr,
task,
user,
vbd,
'vdi-snapshot': vdiSnapshot,
'vdi-unmanaged': vdiUnmanaged,
vdi,
vgpu,
vgpuType,
vif,
'vm-controller': vmController,
'vm-snapshot': vmSnapshot,
'vm-template': vmTemplate,
vm,
vtpm,
} as const

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,9 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
tags: true,
other_config: true,
},
}

View File

@@ -1,5 +0,0 @@
export default {
read: true,
plug: true,
unplug: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
management: true,
},
}

View File

@@ -1,14 +0,0 @@
export default {
'add-host': true,
create: {
network: true,
vm: true,
},
'emergency-shutdown': true,
read: true,
'rolling-reboot': true,
'rolling-update': true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,4 +0,0 @@
export default {
read: true,
run: true,
}

View File

@@ -1,7 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,16 +0,0 @@
export default {
create: true,
delete: true,
forget: true,
import: {
vdi: true,
vm: true,
},
'migrate-receive': true,
read: true,
'reclaim-space': true,
scan: true,
update: {
tags: true,
},
}

View File

@@ -1,5 +0,0 @@
export default {
abort: true,
delete: true,
read: true,
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
password: true,
permission: true,
preferences: true,
},
}

View File

@@ -1,7 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
}

View File

@@ -1,8 +0,0 @@
export default {
delete: true,
export: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,15 +0,0 @@
export default {
boot: true,
create: true,
delete: true,
'export-content': true,
'import-content': true,
'migrate-send': true,
read: true,
update: {
name_description: true,
name_label: true,
size: true,
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,15 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
update: {
allowedIpv4Addresses: true,
allowedIpv6Addresses: true,
lockingMode: true,
rateLimit: true,
txChecksumming: true,
other_config: true,
},
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
tags: true,
},
}

View File

@@ -1,8 +0,0 @@
export default {
delete: true,
export: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,9 +0,0 @@
export default {
delete: true,
export: true,
instantiate: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,61 +0,0 @@
export default {
clone: true,
delete: true,
export: true,
'migrate-send': true,
pause: true,
read: true,
reboot: {
clean: true,
hard: true,
},
resume: true,
'revert-snapshot': true,
shutdown: {
clean: true,
hard: true,
},
snapshot: true,
start: true,
suspend: true,
unpause: true,
update: {
affinityHost: true,
autoPoweron: true,
blockedOperations: true,
coresPerSocket: true,
cpuCap: true,
cpuMask: true,
cpuWeight: true,
cpus: true,
cpusStaticMax: true,
creation: true,
datasources: true,
expNestedHvm: true,
hasVendorDevice: true,
highAvailability: true,
hvmBootFirmware: true,
memory: true,
memoryMax: true,
memoryMin: true,
memoryStaticMax: true,
nameDescription: true,
nameLabel: true,
nestedVirt: true,
nicType: true,
notes: true,
PV_args: true,
resourceSet: true,
secureBoot: true,
share: true,
startDelay: true,
suspendSr: true,
tags: true,
uefiMode: true,
vga: true,
videoram: true,
viridian: true,
virtualizationMode: true,
xenStoreData: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,119 +0,0 @@
import * as CM from 'complex-matcher'
import type { XoAclPrivilege, XoAclSupportedActions, XoAclSupportedResource } from '@vates/types/lib/xen-orchestra/acl'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../actions/index.mjs'
export type SupportedActionsByResource = typeof SUPPORTED_ACTIONS_BY_RESOURCE
export type SupportedResource = XoAclSupportedResource<SupportedActionsByResource>
export type SupportedActions<T extends SupportedResource> = XoAclSupportedActions<SupportedActionsByResource, T>
export type TPrivilege<T extends SupportedResource> = XoAclPrivilege<SupportedActionsByResource, T>
export class Privilege<T extends SupportedResource> {
#action: TPrivilege<T>['action']
#selector?: (object: object) => boolean
#resource: TPrivilege<T>['resource']
#effect: TPrivilege<T>['effect']
constructor({
action,
selector,
resource,
effect,
}: {
action: TPrivilege<T>['action']
selector?: TPrivilege<T>['selector']
resource: TPrivilege<T>['resource']
effect: TPrivilege<T>['effect']
}) {
Privilege.checkActionIsValid(resource, action)
this.#action = action
this.#selector = selector !== undefined ? CM.parse(selector).createPredicate() : undefined
this.#resource = resource
this.#effect = effect
}
get effect() {
return this.#effect
}
#matchAction<Resource extends SupportedResource = T>(action: SupportedActions<Resource>) {
// read:name_label - read:name_label
if (this.#action === action) {
return true
}
const [namespaceToMatch, actionToMatch] = action.split(':')
const [thisNamespace, thisAction] = this.#action.split(':')
if (thisNamespace === '*') {
return true
}
// update:vm - read:name_label
if (thisNamespace !== namespaceToMatch) {
return false
}
// read - read:name_label
if (thisAction === undefined) {
return true
}
// read:name_decription - read:name_label
if (thisAction !== actionToMatch) {
return false
}
throw new Error(`Unable to verify if ${this.#action} match ${action} `)
}
#matchSelector(object: object) {
if (this.#selector === undefined) {
return true
}
return this.#selector(object)
}
#matchResource(resource: string): resource is SupportedResource {
return resource === this.#resource
}
match<Resource extends SupportedResource = T>(constraint: {
action: SupportedActions<Resource>
resource: Resource
object: object
}): boolean {
return (
this.#matchResource(constraint.resource) &&
this.#matchAction(constraint.action) &&
this.#matchSelector(constraint.object)
)
}
/**
* Throw if action not supported
*/
static checkActionIsValid<T extends SupportedResource>(resource: T, action: SupportedActions<T>) {
const supportedActions = SUPPORTED_ACTIONS_BY_RESOURCE[resource]
if (supportedActions === undefined) {
throw new Error(`${resource} resource not supported`)
}
if (action === '*') {
return
}
const segments = action.split(':')
let _action: object | undefined = undefined
segments.forEach(segment => {
_action = (_action ?? supportedActions)[segment]
if (_action === undefined) {
throw new Error(
`${action} action not supported for the resource: ${resource}. See ${JSON.stringify(supportedActions)}`
)
}
})
}
}

View File

@@ -1,10 +0,0 @@
// DO NOT EDIT
// Overwritten by scripts/generate-types.mjs after running npm run build
// scripts/generate-types.mjs will replace this file to automatically generate and export all privileged types.
// This makes external use easier because it's not necessary to resolve the type every time.
import type { Privilege, SupportedResource } from '../index.mjs'
export type AnyPrivilege = {
[Resource in SupportedResource]: Privilege<Resource>
}[SupportedResource]

View File

@@ -1,167 +0,0 @@
import assert from 'node:assert'
import type { XoUser } from '@vates/types/xo'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
import {
Privilege as CPrivilege,
TPrivilege as Privilege,
SupportedActions,
SupportedActionsByResource,
SupportedResource,
} from './class/privilege.mjs'
// Export all types constructed using SUPPORTED_ACTIONS_BY_RESOURCE
export type { Privilege, SupportedActions, SupportedActionsByResource, SupportedResource }
export { SUPPORTED_ACTIONS_BY_RESOURCE }
export type * from './generated/privilege-types.mjs'
import { AnyPrivilege } from './generated/privilege-types.mjs'
export type AnyPrivilegeOnParam = {
[Resource in SupportedResource]: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: object | object[]
}
}[SupportedResource]
export function hasPrivilegeOn<T extends SupportedResource>({
user,
action,
resource,
objects,
userPrivileges,
}: {
user: XoUser
resource: T
action: SupportedActions<T>
objects: object | object[]
userPrivileges: AnyPrivilege[]
}) {
// Function that will be called outside of the module
// We cannot be sure types are respected
assert.strictEqual(typeof user?.permission, 'string')
assert.strictEqual(typeof action, 'string')
assert.strictEqual(typeof resource, 'string')
assert.strictEqual(objects === undefined, false)
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
if (user.permission === 'admin') {
return true
}
const effectsByObject = new Map<object, Privilege<T>['effect'][]>()
if (Array.isArray(objects)) {
objects.forEach(obj => effectsByObject.set(obj, []))
} else {
effectsByObject.set(objects, [])
}
for (const userPrivilege of userPrivileges) {
const privilege = new CPrivilege(userPrivilege as Privilege<typeof userPrivilege.resource>)
for (const [object, effects] of effectsByObject.entries()) {
if (!privilege.match({ action, resource, object })) {
continue
}
effects.push(privilege.effect)
}
}
if (
Array.from(effectsByObject.values()).some(
effects => effects.length === 0 || effects.some(effect => effect === 'deny')
)
) {
return false
}
return true
}
export function getMissingPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return params
.filter(
param =>
!hasPrivilegeOn({
user: param.user,
resource: param.resource,
action: param.action as SupportedActions<typeof param.resource>,
objects: param.objects,
userPrivileges,
})
)
.map(({ action, objects, resource }) => {
let objectIds: unknown[] | undefined
let objectId: unknown | undefined
if (Array.isArray(objects)) {
if (objects.length > 1) {
objectIds = []
objects.forEach(obj => {
if ('id' in obj) {
objectIds!.push(obj.id)
}
})
} else {
objects = objects?.[0] ?? {}
}
}
if (!Array.isArray(objects)) {
objectId = 'id' in objects ? objects.id : undefined
}
return {
objectId,
objectIds,
action,
resource,
}
})
}
export function hasPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return getMissingPrivileges(params, userPrivileges).length === 0
}
export function filterObjectsWithPrivilege<Resource extends SupportedResource, Object extends object>(param: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: Object[]
userPrivileges: AnyPrivilege[]
}) {
return param.objects.filter(obj => hasPrivilegeOn({ ...param, objects: obj }))
}
// Utils
/**
* Mirror GetKeysRecursively from @vates/types
*/
export function getActionStrings<Resource extends SupportedResource, Actions = SupportedActions<Resource>>(
resource: Resource
): Actions[] {
type NestedActions = { [key: string]: boolean | NestedActions }
function collectActions(obj: NestedActions, prefix?: Actions): Actions[] {
const actions: Actions[] = ['*' as Actions]
for (const [key, value] of Object.entries(obj)) {
const action = (prefix !== undefined ? `${prefix}:${key}` : key) as Actions
actions.push(action)
if (typeof value === 'boolean') {
continue
}
actions.push(...collectActions(value, action))
}
return actions
}
return collectActions(SUPPORTED_ACTIONS_BY_RESOURCE[resource])
}

View File

@@ -1,378 +0,0 @@
import { test, suite } from 'node:test'
import assert from 'node:assert'
import type { Branded } from '@vates/types/common'
import type { XoUser } from '@vates/types/xo'
import { Privilege } from '../class/privilege.mjs'
import type { Privilege as TPrivilege } from '../index.mjs'
import {
filterObjectsWithPrivilege,
getActionStrings,
getMissingPrivileges,
hasPrivilegeOn,
hasPrivileges,
} from '../index.mjs'
suite('Privilege.checkActionIsValid behavior', () => {
test('throw on invalid resource', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('foo', 'read'))
})
test('throw on invalid action', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('vm', 'foo:vm'))
})
test('should work', () => {
assert.doesNotThrow(() => Privilege.checkActionIsValid('vm', 'read'))
})
})
suite('Privilege.match behavior', () => {
const privilege = new Privilege({ action: 'read', resource: 'vm', effect: 'allow' })
const object = { id: 1, name_label: 'foo' }
test('Should not match because of the action', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'start', object }), false)
})
test('Should not match because of the resource', () => {
assert.strictEqual(privilege.match({ resource: 'host', action: 'read', object }), false)
})
test('Should match', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'read', object }), true)
})
test('Should match even if the action is not equal to the action privilege', () => {
const shutdownPrivilege = new Privilege({ action: 'shutdown', resource: 'vm', effect: 'allow' })
assert.strictEqual(shutdownPrivilege.match({ resource: 'vm', action: 'shutdown:clean', object }), true)
})
test('Should match because the action is *', () => {
const vmPrivilege = new Privilege({ action: '*', resource: 'vm', effect: 'allow' })
assert.strictEqual(vmPrivilege.match({ action: 'reboot:hard', resource: 'vm', object: object }), true)
})
test('Should not match due to insufficient privilege', () => {
const cleanShutdownPrivilege = new Privilege({ action: 'shutdown:clean', resource: 'vm', effect: 'allow' })
assert.strictEqual(cleanShutdownPrivilege.match({ resource: 'vm', action: 'shutdown', object }), false)
})
})
suite('ACL V2 behavior', async () => {
// === Setup ===
const admin = {
id: '1' as XoUser['id'],
groups: [],
permission: 'admin',
email: 'admin@admin.net',
preferences: {},
} satisfies XoUser
const user = {
id: '2' as XoUser['id'],
groups: [],
permission: 'none',
email: 'user@user.net',
preferences: {},
} satisfies XoUser
const privilegeId = 'fake-priv-test-id' as Branded<'acl-v2-privilege'>
const roleId = 'fake-role-test-id' as Branded<'acl-v2-role'>
// === Objects
const poolWithoutPrivilege = { id: 'pool-id' }
const alpineVm = { id: 'alpine', creation: { creator: admin.id } }
const almaVm = { id: 'alma', creation: { creator: user.id } }
const xoaVm = { id: 'xoa' }
const template = { id: 'template-id', $pool: '1' }
const templateDeny = { id: 'template-deny-id', $pool: '2' }
const vdis = [{ $SR: '2' }, { $SR: '2' }]
const vdisMultipleSrs = [{ $SR: '1' }, { $SR: '2' }]
const vifs = [{ $network: '1' }]
const iso = { id: 'iso-vdi-id', $SR: '3' }
const hostAffinity = { id: 'affinity-host-id', $pool: '1' }
// === DB Privileges
const vmRead = { id: privilegeId, action: 'read', resource: 'vm', effect: 'allow', roleId } satisfies TPrivilege<'vm'>
const vmCleanShutdown = {
id: privilegeId,
action: 'shutdown:clean',
resource: 'vm',
selector: `creation:creator:${user.id}`,
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm'>
const xoaVmDenyRead = {
id: privilegeId,
action: 'read',
resource: 'vm',
effect: 'deny',
selector: `id: ${xoaVm.id}`,
roleId,
} satisfies TPrivilege<'vm'>
const vdiCreate = {
id: privilegeId,
action: 'create',
resource: 'vdi',
selector: '$SR:2',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const vifCreate = {
id: privilegeId,
action: 'create',
resource: 'vif',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vif'>
const templateInstantiate = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm-template'>
const templateInstantiateDeny = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
selector: '$pool:2',
effect: 'deny',
roleId,
} satisfies TPrivilege<'vm-template'>
const isoVdiUse = {
id: privilegeId,
action: 'boot',
resource: 'vdi',
selector: '$SR:3',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const hostVm = {
id: privilegeId,
action: 'allow-vm',
resource: 'host',
effect: 'allow',
roleId,
} satisfies TPrivilege<'host'>
const allPrivileges = [
vmRead,
vmCleanShutdown,
xoaVmDenyRead,
vdiCreate,
vifCreate,
templateInstantiate,
templateInstantiateDeny,
isoVdiUse,
hostVm,
]
// === Test ===
suite('hasPrivilegeOn behavior', () => {
test('Should always return true for admin', () => {
assert.strictEqual(
hasPrivilegeOn<'pool'>({
user: admin,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
true
)
})
test('Should return false by default for non admin', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
false
)
})
test('Should return true if user has privilege', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: alpineVm, userPrivileges: allPrivileges }),
true
)
})
test('Should return false if user have a both deny and allow privileges', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: xoaVm, userPrivileges: allPrivileges }),
false
)
})
test('Should handle multiple and single object', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm],
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm, xoaVm],
userPrivileges: allPrivileges,
}),
false
)
})
test('Should allow clean shutdown only own VMs', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: almaVm,
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: [almaVm, alpineVm],
userPrivileges: allPrivileges,
}),
false
)
})
})
suite('getMissingPrivileges behavior', () => {
test('Should return empty array if no missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'shutdown:clean', resource: 'vm', objects: almaVm },
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 0)
})
test('Should return only missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege },
{ user, action: '*', resource: 'pool', objects: poolWithoutPrivilege },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 2)
})
test('Should return objectId if only one object in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectId, undefined)
})
test('Should return objectIds if multiple objects in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: [poolWithoutPrivilege, poolWithoutPrivilege] }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectIds, undefined)
})
})
suite('hasPrivileges behavior', () => {
test('Should return false if at least one privilege deny', () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: templateDeny },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
test("Should return false if at least one privilege didn't match", () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdisMultipleSrs },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
})
suite('filterObjectsWithPrivilege behavior', () => {
test('Should return only objects that match privileges', () => {
let vms = [almaVm, alpineVm, xoaVm, xoaVm]
assert.strictEqual(vms.length, 4)
vms = filterObjectsWithPrivilege({
user,
action: 'read',
resource: 'vm',
objects: vms,
userPrivileges: allPrivileges,
})
assert.strictEqual(vms.length, 2)
})
})
})
suite('getActionStrings', () => {
test('returns flat actions for a resource without nesting', () => {
const actions = getActionStrings('host')
assert.ok(actions.includes('read'))
// @ts-ignore A TypeScript error is thrown because `foo` is not of type `SupportedActions<'vm'>`.
// To test for a negative value, the error is ignored.
assert.strictEqual(actions.includes('foo'), false)
})
test('parent action is included alongside its children', () => {
const actions = getActionStrings('vm')
assert.ok(actions.includes('shutdown'))
assert.ok(actions.includes('shutdown:clean'))
assert.ok(actions.includes('shutdown:hard'))
})
})

View File

@@ -1,19 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"module": "nodenext",
"lib": ["ESNext"],
"moduleResolution": "nodenext",
"target": "ES2022",
"skipLibCheck": true,
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
"noImplicitAny": false, // otherwise it will throw an error when importing packages without type definition
"noUncheckedIndexedAccess": true,
"declaration": true
},
"include": ["./src/**/*"],
"exclude": ["./node_modules", "./dist"]
}

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/async-map",
"version": "0.1.3",
"version": "0.1.2",
"license": "ISC",
"description": "Promise.all + map for all iterables",
"keywords": [
@@ -28,7 +28,7 @@
"node": ">=6"
},
"dependencies": {
"lodash": "^4.18.0"
"lodash": "^4.17.4"
},
"scripts": {
"postversion": "npm publish",

View File

@@ -17,7 +17,7 @@
},
"dependencies": {
"@vates/decorate-with": "^2.1.0",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/log": "^0.7.1",
"golike-defer": "^0.5.1",
"object-hash": "^2.0.1"
},

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,21 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
},
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,56 +0,0 @@
{
"name": "@xen-orchestra/backup-archive",
"version": "1.1.1",
"description": "Helper to handle generic backup archives",
"license": "ISC",
"author": "",
"type": "module",
"main": "./dist/index.mjs",
"exports": {
".": "./dist/index.mjs",
"./disks": "./dist/disks/index.mjs"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^6.0.1",
"@types/node": "^22.3.0",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0",
"fs-extra": "^11.1.0",
"sinon": "^18.0.0",
"tar-stream": "^3.1.8",
"@types/tar-stream": "^3.1.4",
"tmp": "^0.2.1",
"@vates/types": "^1.30.1"
},
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/fs": "^4.9.3",
"@xen-orchestra/disk-transform": "^1.3.3",
"@vates/task": "0.7.0",
"golike-defer": "0.5.1",
"uuid": "^9.0.0",
"promise-toolbox": "^0.21.0",
"@vates/async-each": "^1.0.1",
"vhd-lib": "^4.16.1"
},
"scripts": {
"build": "yarn run clean && tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "yarn run clean && tsc && node --test **/*.test.mjs",
"test-integration": "yarn run clean && tsc && node --test **/*.integ.mjs",
"clean": "rimraf dist/"
},
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/backup-archive",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20.17"
}
}

View File

@@ -1,438 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import {
MergeRemoteDisk,
openDisk,
openDisposableDisk,
RemoteDisk,
isDisk,
openDiskChain,
} from '@xen-orchestra/backup-archive/disks'
import Disposable from 'promise-toolbox/Disposable'
import {
CleanResult,
DEFAULT_MERGE_CONCURRENCY,
DEFAULT_REMOVE_CONCURRENCY,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { asyncEach } from '@vates/async-each'
import { Task } from '@vates/task'
import { limitConcurrency } from 'limit-concurrency-decorator'
import assert from 'node:assert/strict'
const defaultMergeLimiter = limitConcurrency(1)
const MERGE_STATE_FILE_RE = /^\.(.+)\.merge\.json$/
/**
* Tracks the disk chain for a single VDI across all backup snapshots.
* Owns merge and deletion decisions for its chain given which disks are still
* referenced by active backups (accumulated via addActiveDiskPath).
*/
export class RemoteDiskLineage {
#handler: RemoteHandlerAbstract
#vdiDir: string
#opts: ResolvedBackupCleanOptions
// Concrete disk paths discovered in this VDI directory (normalized)
#diskPaths: Set<string> = new Set()
// Disk paths that exist on disk but failed to open (corrupted)
#brokenDiskPaths: Set<string> = new Set()
// child path: parent path (from disk header)
#parentOf: Map<string, string> = new Map()
// parent path: child path
#childOf: Map<string, string> = new Map()
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
#activeDiskPaths: Set<string> = new Set()
// Interrupted merges: normalized parent path { stateFilePath, chain }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
this.#handler = handler
this.#vdiDir = vdiDir
this.#opts = opts
}
#unregisterDisk(diskPath: string): void {
this.#diskPaths.delete(diskPath)
const parent = this.#parentOf.get(diskPath)
if (parent !== undefined) {
this.#parentOf.delete(diskPath)
this.#childOf.delete(parent)
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
this.#childOf.delete(diskPath)
this.#parentOf.delete(child)
}
}
/**
* Lists disk files and reads their headers to build the parent-child chain.
* Also detects interrupted merge state files.
*/
async init(): Promise<void> {
let files: string[]
try {
files = await this.#handler.list(this.#vdiDir, { prependDir: true })
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.#opts.logWarn('failed to list VDI directory', { vdiDir: this.#vdiDir, error })
return
}
for (const filePath of files) {
if (isDisk(filePath)) {
this.#diskPaths.add(normalize(filePath))
}
}
const mergeParentPaths = new Set<string>()
for (const filePath of files) {
const match = MERGE_STATE_FILE_RE.exec(basename(filePath))
if (match !== null) {
mergeParentPaths.add(normalize(this.#vdiDir + '/' + match[1]))
}
}
const uuidToPath = new Map<string, string>()
for (const diskPath of this.#diskPaths) {
let disk: RemoteDisk | undefined
try {
if (mergeParentPaths.has(diskPath)) {
disk = await openDisk({ handler: this.#handler, path: diskPath, force: true })
} else {
disk = await openDisk({ handler: this.#handler, path: diskPath })
}
} catch (error) {
if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
throw error
}
this.#opts.logWarn('failed to open disk', { path: diskPath, error })
// Broken disks will be removed by the clean phase
// because they are marked as NOT active
this.#brokenDiskPaths.add(diskPath)
continue
}
try {
const uuid = disk.getUuid()
// Detect Disks with the same UUIDs
// Due to a bug introduced in a1bcd35e2
const existingPath = uuidToPath.get(uuid)
if (existingPath !== undefined) {
this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
let existingDisk: RemoteDisk | undefined
try {
existingDisk = await openDisk({ handler: this.#handler, path: existingPath })
if (existingDisk.containsAllDataOf(disk)) {
this.#opts.logWarn('should drop duplicate disk, existing is superset', { dropped: diskPath })
continue
} else if (disk.containsAllDataOf(existingDisk)) {
this.#opts.logWarn('should drop old duplicate disk, new is superset', { dropped: existingPath })
} else {
this.#opts.logWarn('duplicate disks have different content, should first-seen', { dropped: diskPath })
continue
}
} catch (error) {
this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
this.#unregisterDisk(existingPath)
} finally {
await existingDisk?.close()
}
}
uuidToPath.set(uuid, diskPath)
if (disk.isDifferencing()) {
const parentPath = disk.getParentPath()
this.#parentOf.set(diskPath, parentPath)
if (this.#childOf.has(parentPath)) {
const error = new Error('this script does not support multiple children', {
cause: {
parent: parentPath,
child1: this.#childOf.get(parentPath),
child2: diskPath,
},
})
throw error
}
this.#childOf.set(parentPath, diskPath)
}
} catch (error) {
this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
} finally {
await disk.close()
}
}
this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
this.#handler,
this.#vdiDir,
files,
uuidToPath,
this.#childOf
)
const propagateBroken = (diskPath: string) => {
if (!this.#brokenDiskPaths.has(diskPath)) {
this.#brokenDiskPaths.add(diskPath)
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
propagateBroken(child)
}
}
}
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent) || this.#brokenDiskPaths.has(parent)) {
this.#opts.logWarn('disk broken: parent missing or broken', { child, parent })
propagateBroken(child)
}
}
}
/**
* Returns true if the disk path exists on disk and opened without errors during init.
* Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
*/
isPathAccessible(diskPath: string): boolean {
return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
}
/** Returns true if the disk exists but failed to open during init. */
isBroken(diskPath: string): boolean {
return this.#brokenDiskPaths.has(diskPath)
}
/** Verifies that every parent referenced in disk headers exists in the directory. */
async check(): Promise<void> {
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent)) {
this.#opts.logWarn('parent disk is missing', { child, parent })
}
}
}
/**
* Declares disk paths as active for this lineage.
* Called by each archive that references this lineage after it determines it is complete.
* Accumulated across all referencing archives before clean() is called.
*/
addActiveDiskPath(diskPath: string): void {
this.#activeDiskPaths.add(normalize(diskPath))
}
/**
* For each orphan disk:
* - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
* - If it has no active descendant → delete it (if opts.remove).
*
* Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
* Deletes orphan merge state files (for missing disks) when opts.remove is true.
*
* Requires activeDiskPaths to be completed to have been called by all referencing archives before this.
* @returns Set of deleted disk paths.
*/
async clean({
remove = this.#opts.remove ?? false,
merge = this.#opts.merge ?? false,
}: { remove?: boolean; merge?: boolean } = {}): Promise<CleanResult> {
const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
const toDelete = new Set<string>()
const toMerge: { chain: string[]; isResuming: boolean }[] = []
const visited = new Set<string>()
// Walk from an orphan toward its descendants.
// Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
// or undefined if the subtree is entirely orphaned (mark for deletion).
const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
if (visited.has(diskPath)) {
return undefined
}
visited.add(diskPath)
if (!orphanDisks.has(diskPath)) {
// Active disk, anchor of the merge chain
return [diskPath]
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
const chain = getUsedChildChainOrDelete(child)
if (chain !== undefined) {
chain.unshift(diskPath)
return chain
}
}
toDelete.add(diskPath)
return undefined
}
// Process interrupted merges first so their disks are protected from the orphan loop
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
if (!this.#diskPaths.has(parentPath)) {
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
if (remove) {
this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
let chain: string[] | undefined
if (stateChain !== undefined) {
const existing = stateChain.filter(p => this.#diskPaths.has(p))
if (existing.length >= 2) chain = existing
}
if (chain !== undefined) {
chain.forEach(p => visited.add(p))
toMerge.push({ chain, isResuming: true })
}
}
for (const orphan of orphanDisks) {
if (!visited.has(orphan)) {
const parent = this.#parentOf.get(orphan)
if (parent === undefined || !orphanDisks.has(parent)) {
const chain = getUsedChildChainOrDelete(orphan)
if (chain !== undefined && chain.length > 1) {
toMerge.push({ chain, isResuming: false })
}
}
}
}
if (remove) {
await asyncEach(
Array.from(toDelete),
async path => {
this.#opts.logInfo('deleting unused disk', { path })
try {
const disk = await openDisk({ handler: this.#handler, path, ignoreBlockIndexes: true })
await disk.unlink({ force: true })
// Remove from tracking so #cleanOrphanDataFiles skips this now-deleted path
this.#unregisterDisk(path)
} catch (error) {
this.#opts.logWarn('failed to delete unused disk', { path, error })
}
},
{ concurrency: DEFAULT_REMOVE_CONCURRENCY }
)
}
if (toMerge.length > 0) {
this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
}
// mergeTargetPath: final size of the disk everything was merged into
const mergedSizes = new Map<string, number>()
if (merge) {
const limitedMergeChain = defaultMergeLimiter(this.#mergeChain.bind(this))
const doMerge = async () => {
await asyncEach(
toMerge,
async ({ chain, isResuming }) => {
const { finalDiskSize, mergeTargetPath } = await limitedMergeChain([...chain], isResuming)
mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
// parentPath alias deleted by parentDisk.rename(mergeTargetPath)
// intermediates deleted by childDisk.unlink() when removeUnused=true
// Unregister so #cleanOrphanDataFiles does not read already-deleted aliases
// and produce false "missing target of alias" warnings.
for (const deletedPath of chain.slice(0, -1)) {
this.#unregisterDisk(deletedPath)
}
},
{ concurrency: this.#opts.mergeConcurrency ?? DEFAULT_MERGE_CONCURRENCY }
)
}
if (toMerge.length !== 0) {
await Task.run({ properties: { name: 'merge' } }, doMerge)
}
}
await this.#cleanOrphanDataFiles(remove)
return {
removedFiles: Array.from(toDelete),
merge: toMerge.length > 0,
size: [...mergedSizes.values()].reduce((total, subSize) => total + subSize, 0),
mergedSizes,
}
}
/**
* Merges ancestors into the active child at the end of the chain.
* Returns the final size of the merge target and its path.
*/
async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
assert.ok(chain.length >= 2, `look to merge a chain shorter than 2 ${JSON.stringify(chain)}`)
const parentPath = chain[0]
this.#opts.logInfo('merging disk chain', { chain: [...chain] }) // need a copy here to log full chain
// The last disk in the chain is the active one that everything gets merged into
const mergeTargetPath = chain.pop()!
const merger = new MergeRemoteDisk(this.#handler, {
logInfo: this.#opts.logInfo,
removeUnused: this.#opts.remove,
mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
onProgress: this.#opts.onProgress,
})
let parentDisk: RemoteDisk | undefined
let childDisk: RemoteDisk | undefined
try {
parentDisk = await openDisk({ handler: this.#handler, path: parentPath, force: isResuming })
childDisk = await openDiskChain({
handler: this.#handler,
path: mergeTargetPath,
until: parentPath,
force: isResuming,
})
const { finalDiskSize } = (await merger.merge(parentDisk, childDisk!))!
return { finalDiskSize, mergeTargetPath }
} finally {
await parentDisk?.close().catch(error => {
this.#opts.logWarn('Error closing parent Disk', error)
})
await childDisk?.close().catch(error => {
this.#opts.logWarn('Error closing child Disk', error)
})
}
}
/**
* Validates each disk integrity, then scans discovered data subdirs
* and deletes any files not associated to a known disk.
*/
async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
const claimedFiles = new Set<string>()
for (const diskPath of this.#diskPaths) {
const claimed = await Disposable.use(
openDisposableDisk({ handler: this.#handler, path: diskPath, ignoreBlockIndexes: true }),
async disk => {
await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
return disk.listAssociatedFiles(this.#vdiDir)
}
)
for (const f of claimed) claimedFiles.add(normalize(f))
}
const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
for (const dataDir of dataDirs) {
const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
for (const item of items) {
if (!claimedFiles.has(normalize(item))) {
this.#opts.logWarn('orphaned data file', { path: item })
if (remove) {
await this.#handler.unlink(item).catch(async err => {
if (err?.code === 'EISDIR') {
await this.#handler.rmtree(item)
} else {
this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
}
})
}
}
}
}
}
}

View File

@@ -1,74 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
export interface PartialBackupMetadata {
mode: 'full' | 'delta'
xva?: string
vhds?: Record<string, string>
vdis?: Record<string, unknown>
isVhdDifferencing?: Record<string, boolean>
size?: number
jobId: string
scheduleId: string
timestamp: number
// 'true' when written by code including the non-NBD qcow2 corruption fix and its force-full
// gate; its absence marks a potentially-corrupt older backup and makes the gate re-probe.
// Preserved (never stamped) across merges here.
// @todo remove this code and the logic associated on 2027 01 01
includeNonNbdQcow2Fix?: boolean
}
export const DEFAULT_MERGE_CONCURRENCY = 1
export const DEFAULT_REMOVE_CONCURRENCY = 4
export interface BackupCleanOptions {
fix?: boolean
merge?: boolean
remove?: boolean
mergeBlockConcurrency?: number
mergeConcurrency?: number
onProgress?: (progress: { total: number; done: number }) => void
logInfo?: (message: any, opts?: object) => void
logWarn?: (message: any, opts?: object) => void
}
export type ResolvedBackupCleanOptions = BackupCleanOptions & {
logInfo: (message: any, opts?: object) => void
logWarn: (message: any, opts?: object) => void
}
export interface ArchiveCleanOptions {
remove?: boolean
merge?: boolean
mergedSizes?: Map<string, number>
}
export interface CheckResult {
isValid: boolean
// Files referenced by metadata but absent on disk
missingDisks?: string[]
// Disk files that exist but failed to open (corrupted)
brokenDisks?: string[]
// Files present on disk but not referenced by metadata
orphans?: string[]
// Files referenced by at least one valid backup
linked?: string[]
}
export type CleanResult = {
removedFiles: string[]
merge?: boolean
size?: number
mergedSizes?: Map<string, number>
}
export interface VmBackupInterface {
handler: RemoteHandlerAbstract
// metadataPath: string
// metadata: PartialBackupMetadata
rootPath: string
opts: BackupCleanOptions
init(): Promise<void>
check(): Promise<CheckResult>
clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
getAssociatedFiles(opts: object): Array<string>
}

View File

@@ -1,289 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, normalize } from '@xen-orchestra/fs/path'
import { resolve } from 'node:path'
import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
import {
ArchiveCleanOptions,
BackupCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
DEFAULT_MERGE_CONCURRENCY,
} from './VmBackup.types.mjs'
import { cleanOrphanDiskDirs } from '@xen-orchestra/backup-archive/disks'
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { promisify } from 'node:util'
import zlib from 'node:zlib'
const gzip = promisify(zlib.gzip)
const gunzip = promisify(zlib.gunzip)
const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
export class VmBackupDirectory implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
files: Array<string> = new Array()
orphans: Set<string> = new Set()
backupArchives: Map<string, VmBackupInterface> = new Map()
opts: ResolvedBackupCleanOptions
// Cached result of the last check() call; invalidated by init()
#checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
#uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
// Set by #checkCacheCount(): the on-disk cache did not match the archives found
// on disk, so clean() must regenerate it even when nothing was merged/removed.
#cacheOutOfSync = false
constructor(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions = {
fix: true,
merge: false,
remove: false,
logInfo,
logWarn,
}
) {
this.handler = handler
this.rootPath = vmBackupPath
this.opts = {
...opts,
fix: opts.fix ?? true,
merge: opts.merge ?? false,
remove: opts.remove ?? false,
logInfo: opts.logInfo ?? logInfo,
logWarn: opts.logWarn ?? logWarn,
}
}
async #readCache(path: string): Promise<Record<string, unknown> | undefined> {
try {
return JSON.parse((await gunzip(await this.handler.readFile(path))).toString())
} catch (error) {
if (error?.code !== 'ENOENT') {
logWarn('failed to read cache', { error, path })
}
}
}
async #writeCache(path: string, data: Record<string, unknown>): Promise<void> {
try {
await this.handler.writeFile(path, await gzip(JSON.stringify(data)), { flags: 'w' })
} catch (error) {
logWarn('failed to write cache', { error, path })
}
}
async init() {
this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
this.#checkResult = undefined
this.#uniqueLineages = new Map()
for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
let metadata: PartialBackupMetadata | undefined = undefined
try {
metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
} catch (error) {
this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
}
if (metadata !== undefined) {
try {
const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
this.backupArchives.set(fullPath, backupArchive)
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
}
}
}
}
getAssociatedFiles({ prefix = false }) {
const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
return prefix ? files : files.map(file => basename(file))
}
async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
await this.#checkCacheCount()
for (const backupArchive of this.backupArchives.values()) {
await backupArchive.check()
}
// allUsedFiles is used for root-level orphan detection only.
// Active disk paths are accumulated per-lineage by each archive during check() above.
const allUsedFiles = new Set<string>([
...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
...this.getAssociatedFiles({ prefix: true }),
])
for (const lineage of this.#uniqueLineages!.values()) {
await lineage.check()
}
const orphans = this.files.filter(file => !allUsedFiles.has(file))
const linked = Array.from(allUsedFiles)
this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
return this.#checkResult
}
async clean({
remove = this.opts.remove ?? false,
merge = this.opts.merge ?? false,
}: ArchiveCleanOptions = {}): Promise<CleanResult> {
// Use cached check result if available, otherwise run check now
const { orphans } = this.#checkResult ?? (await this.check())
let cacheNeedsRegen = false
let someLineageMergedOrShouldBe = false
// Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
const allMergedSizes = new Map<string, number>()
await asyncEach(
Array.from(this.#uniqueLineages!.entries()),
async ([_vdiDir, lineage]) => {
const { mergedSizes, removedFiles, merge: hasPendingMerge } = await lineage.clean({ remove, merge })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
if (hasPendingMerge) {
someLineageMergedOrShouldBe = true
}
for (const [diskPath, size] of mergedSizes || []) {
allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
}
},
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
)
// Delete VDI directories not referenced by any archive
const coveredDirs = new Set(this.#uniqueLineages!.keys())
await cleanOrphanDiskDirs(this.handler, this.rootPath, coveredDirs, {
remove,
logWarn: this.opts.logWarn,
logInfo: this.opts.logInfo,
})
// Let each archive clean its own files (e.g. remove metadata for incomplete backups)
// and update metadata with merged sizes if applicable
await asyncEach(
Array.from(this.backupArchives.values()),
async (archive: VmBackupInterface) => {
const { removedFiles } = await archive.clean({ remove, mergedSizes: allMergedSizes })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
},
{ concurrency: 2 }
)
if (allMergedSizes.size > 0 || cacheNeedsRegen || this.#cacheOutOfSync) {
await this.#regenerateCache()
}
// Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
if (remove) {
await asyncEach(
orphans,
async (orphan: string) => {
try {
await this.handler.unlink(orphan)
} catch (error) {
if (error?.code === 'EISDIR') {
this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
} else if (error?.code !== 'ENOENT') {
throw error
}
}
},
{ concurrency: 2 }
)
}
const size = [...allMergedSizes.values()].reduce((total, merged) => total + merged, 0)
return { removedFiles: orphans, merge: someLineageMergedOrShouldBe, size: size }
}
async #checkCacheCount(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const existingCache = await this.#readCache(cachePath)
const actual = existingCache === undefined ? 0 : Object.keys(existingCache).length
const expected = this.backupArchives.size
this.#cacheOutOfSync = actual !== expected
if (this.#cacheOutOfSync) {
this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
}
}
async #regenerateCache(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const cache: Record<string, object> = {}
for (const [path, archive] of this.backupArchives.entries()) {
const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
cache[path] = { _filename: path, id: path, ...a.metadata }
}
await this.#writeCache(cachePath, cache)
}
async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
let backupArchive: VmBackupInterface
try {
if (metadata.mode === 'full') {
backupArchive = new VmFullBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
resolve('/', this.rootPath, metadata.xva!),
this.opts
)
} else if (metadata.mode === 'delta') {
const rawDiskPaths = metadata.vhds
const diskPaths =
rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
backupArchive = new VmIncrementalBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
diskPaths,
this.opts,
this.#uniqueLineages
)
} else {
throw new Error(`Mode ${metadata.mode} not supported`)
}
} catch (error) {
this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
}
await backupArchive.init()
return backupArchive
}
/**
* Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
* The `lock` option is accepted but ignored (locking is the caller's responsibility).
*/
static async cleanVm(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions & { lock?: boolean } = {}
) {
const { lock: _lock, ...cleanOpts } = opts
const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
await dir.init()
await dir.check()
const { merge, size } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
return { merge, size }
}
}

View File

@@ -1,157 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
import { rimraf } from 'rimraf'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import tar from 'tar-stream'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, vmBackupDir
const vmUuid = 'test-vm-uuid'
const rootPath = `xo-vm-backups/${vmUuid}`
async function createMinimalXva() {
const pack = tar.pack()
pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
pack.finalize()
const chunks = []
for await (const chunk of pack) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
await fs.mkdirp(`${tempDir}/${rootPath}`)
vmBackupDir = new VmBackupDirectory(handler, rootPath)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
async function createFullBackupMetadata(name, xvaName) {
await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
const size = await handler.getSize(`${rootPath}/${xvaName}`)
const metadata = {
mode: 'full',
xva: `./${xvaName}`,
vm: { uuid: vmUuid },
jobId: uniqueId(),
timestamp: Date.now(),
size,
}
await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
return metadata
}
describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
test('init() loads full backup archives from metadata files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await createFullBackupMetadata('backup2.json', 'backup2.xva')
await vmBackupDir.init()
assert.equal(vmBackupDir.backupArchives.size, 2)
})
test('getAssociatedFiles() returns cache files', async () => {
// Create a cache file
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
await vmBackupDir.init()
const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
assert.equal(filesWithPrefix.length, 1)
assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
})
test('check() identifies orphan files', async () => {
// Create a valid backup
await createFullBackupMetadata('backup1.json', 'backup1.xva')
// Create orphan files (not referenced by any metadata)
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
await vmBackupDir.init()
const { orphans, linked } = await vmBackupDir.check()
assert.ok(orphans.some(f => f.includes('orphan.xva')))
assert.ok(orphans.some(f => f.includes('random-file.txt')))
assert.ok(linked.some(f => f.includes('backup1.json')))
assert.ok(linked.some(f => f.includes('backup1.xva')))
})
test('clean() removes orphan files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.json'))
assert.ok(remainingFiles.includes('backup1.xva'))
assert.ok(!remainingFiles.includes('orphan.xva'))
})
test('Missing XVA cleans the metadata', async () => {
await createFullBackupMetadata('metadata.json', 'missing.xva')
await handler.unlink(`${rootPath}/missing.xva`)
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
const remainingFiles = await handler.list(rootPath)
assert.ok(logged.match(/XVA is missing/g))
assert.ok(!remainingFiles.includes('metadata.json'))
})
test('clean() preserves cache.json.gz', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
await VmBackupDirectory.cleanVm(handler, rootPath)
const remainingFiles = await handler.list(rootPath)
assert.equal(remainingFiles.length, 3)
assert.ok(remainingFiles.includes('cache.json.gz'))
})
test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
})
test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
assert.ok(vmBackupDir.backupArchives.get(`/${rootPath}/backup1.json`).isValid)
assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept')
})
})

View File

@@ -1,179 +0,0 @@
import { basename, normalize } from '@xen-orchestra/fs/path'
import assert from 'node:assert'
import { FileDescriptor } from '@xen-orchestra/fs'
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
// https://tools.ietf.org/html/rfc1952.html#page-5
Buffer.from('1F8B', 'hex'),
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
Buffer.from('28B52FFD', 'hex'),
]
const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
return true
}
}
return false
}
const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
if (size <= 1024 || size % 512 !== 0) {
return false
}
const buf = Buffer.allocUnsafe(1024)
assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
return buf.every(byte => byte === 0)
}
export async function isValidXva(
handler: RemoteHandlerAbstract,
filePath: string,
metadataSize?: number
): Promise<boolean> {
// size is longer when encrypted + reading part of an encrypted file is not implemented
if (handler.isEncrypted) {
const size = await handler.getSizeOnDisk(filePath)
return size > 20
}
let isValid = true
const fd = await handler.openFile(filePath, 'r')
try {
const size = await handler.getSize(filePath)
if (size < 20) {
// neither a valid gzip nor tar
return false
}
let validSize = true
if (metadataSize !== undefined) {
validSize = metadataSize === size
}
const validDisk = (await isCompressedFile(handler, fd))
? true // compressed files cannot be validated at this time
: await isValidTar(handler, size, fd)
isValid = validSize && validDisk
} finally {
handler.closeFile(fd).catch(noop)
}
return isValid
}
const noop = (): void => {}
export class VmFullBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
xvaPath: string
isValid?: boolean
missingDisk = false
metadataPath: string
metadata: PartialBackupMetadata
opts: ResolvedBackupCleanOptions
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
opts: ResolvedBackupCleanOptions
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.xvaPath = normalize(xvaPath)
this.opts = opts
}
async init() {}
/**
* As we are not 100% sure if XVA is valid,
* we prefer to warn if it is the case but not remove it.
*/
async check(): Promise<CheckResult> {
if (this.isValid === undefined) {
try {
this.isValid = await isValidXva(this.handler, this.xvaPath, this.metadata.size)
} catch (error) {
this.isValid = false
if (error?.code === 'ENOENT') {
this.missingDisk = true
}
this.opts.logWarn('Issue while checking XVA', { error })
}
if (!this.handler.isEncrypted) {
try {
await this.handler.getSize(`${this.xvaPath}.checksum`)
} catch (error) {
this.opts.logWarn('Checksum file not valid, not blocking', { error })
}
}
}
if (!this.isValid) {
if (this.missingDisk) {
this.opts.logWarn('XVA is missing', { path: this.xvaPath })
} else {
this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
}
}
return { isValid: this.isValid }
}
async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (this.isValid === undefined) {
await this.check()
}
const removedFiles: string[] = []
if (!this.isValid && this.missingDisk) {
removedFiles.push(this.metadataPath, `${this.xvaPath}.checksum`)
}
if (remove) {
if (!this.isValid) {
if (!this.missingDisk) {
this.opts.logWarn(`This files may be corrupted but not yet to be removed`, {
files: this.getAssociatedFiles({ prefix: false }),
})
} else {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
}
}
return { removedFiles, merge: false }
}
getAssociatedFiles({ prefix = false }): Array<string> {
let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
return prefix ? validFiles : validFiles.map(file => basename(file))
}
}

View File

@@ -1,859 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import { RemoteVhdDisk } from '../dist/disks/RemoteVhdDisk.mjs'
import { MergeRemoteDisk } from '../dist/disks/MergeRemoteDisk.mjs'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
import { dirname, basename } from 'node:path'
import { promisify } from 'node:util'
import zlib from 'node:zlib'
import { rimraf } from 'rimraf'
const { beforeEach, afterEach, describe } = test
const gunzip = promisify(zlib.gunzip)
let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
const rootPath = 'xo-vm-backups/VMUUID/'
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
jobId = uniqueId()
vdiId = uniqueId()
relativePath = `vdis/${jobId}/${vdiId}`
basePath = `${rootPath}/${relativePath}`
vdiId2 = uniqueId()
relativePath2 = `vdis/${jobId}/${vdiId2}`
basePath2 = `${rootPath}/${relativePath2}`
await fs.mkdirp(`${tempDir}/${basePath}`)
await fs.mkdirp(`${tempDir}/${basePath2}`)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
async function generateVhd(path, opts = {}) {
let vhd
let dataPath = path
if (opts.useAlias) {
await handler.mkdir(dirname(path) + '/data/')
dataPath = dirname(path) + '/data/' + basename(path)
}
if (opts.mode === 'directory') {
await handler.mkdir(dataPath)
vhd = new VhdDirectory(handler, dataPath)
} else {
const fd = await handler.openFile(dataPath, 'wx')
vhd = new VhdFile(handler, fd)
}
vhd.header = { ...VHDHEADER, ...opts.header }
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
if (vhd.header.parentUuid) {
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
if (opts.useAlias === true) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
if (opts.blocks) {
for (const blockId of opts.blocks) {
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
}
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
return vhd
}
test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
// metadata references a non-existent disk so the lineage is created for this vdiDir
// and notReallyAVhd.vhd is treated as an unreferenced orphan
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
assert.ok(logged.includes('failed to open disk'))
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent, should be deleted
await generateVhd(`${basePath}/abandoned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// one orphan, which is a full vhd, no parent : should stay
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan in the metadata : should stay
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
})
)
// clean
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
// abandoned has a missing parent, orphan and child have no active backup reference
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
assert.equal(detectedMissingParent.length, 1)
// we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
})
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.json is not here
],
})
)
await generateVhd(`${basePath}/abandoned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
let matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.vhd is not here anymore
],
}),
{ flags: 'w' }
)
logged = ''
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after metadata deletion'
)
})
test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
// A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
// Metadata references the corrupt VHD as if it were a valid active disk
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/broken.vhd`],
})
)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
logInfo: () => {},
logWarn: logInfo,
})
assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
// metadata removed because backup is incomplete
assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
// broken VHD removed via the orphan path in lineage.clean()
assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
})
test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
// Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
// Each snapshot is the active disk for exactly one backup metadata file.
//
// Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
// own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
// RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
// their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
// deleted — destroying two live backups.
const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
})
await generateVhd(`${basePath}/snapshot3.vhd`, {
header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
})
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata3.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
assert.equal(remaining.length, 3, 'no extra files should remain')
// All three metadata files must also survive (all backups are complete)
const rootFiles = await handler.list(rootPath)
assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
})
test('it handles multiple VDI directories independently', async () => {
// vdiDir1: diskA1 (base) <- diskA2, plus an orphan
const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
await generateVhd(`${basePath}/diskA2.vhd`, {
header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
})
await generateVhd(`${basePath}/orphanA.vhd`)
// vdiDir2: diskB1 (base) <- diskB2, plus an orphan
const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
await generateVhd(`${basePath2}/diskB2.vhd`, {
header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
})
await generateVhd(`${basePath2}/orphanB.vhd`)
// archive1 covers both base disks, archive2 covers both child disks
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining1 = await handler.list(basePath)
assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
const remaining2 = await handler.list(basePath2)
assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
})
test('it merges delta of non destroyed chain', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000, // a size too small
vhds: [
`${relativePath}/grandchild.vhd`, // grand child should not be merged
`${relativePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
blocks: [0, 1],
})
// a grand child
await generateVhd(`${basePath}/grandchild.vhd`, {
header: {
parentUnicodeName: 'child.vhd',
parentUuid: child.footer.uuid,
},
})
let logged = []
const logInfo = message => {
logged.push(message)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
assert.equal(logged[0], `unexpected number of entries in backup cache`)
logged = []
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
assert.equal(logged.includes(`merging disk chain`), true)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 2)
assert.equal(remainingVhds.includes('child.vhd'), true)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
assert.equal(metadata.size, 4299776, 'metadata.size should be updated after merge')
})
test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
// ancestor -> child -> grandchild
// only grandchild is referenced in metadata: both ancestor and child are orphans
// they must be merged as a single 3-disk chain, not processed separately
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000,
vhds: [`${relativePath}/grandchild.vhd`],
})
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
const child = await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
})
await generateVhd(`${basePath}/grandchild.vhd`, {
header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
})
const mergeChains = []
const logInfo = (message, opts) => {
if (message === 'merging disk chain') mergeChains.push(opts.chain)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
// all three disks must be handled in exactly one merge, not two separate operations
assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
assert.equal(remainingVhds.includes('child.vhd'), false)
})
test('it resumes, through cleanVm, an interrupted merge whose ancestor lost its footer', async () => {
// Reproduces the production "footer1 !== footer2" failure end-to-end via the full cleanVm
// pipeline. ancestor (full) <- child (differencing, the active backup).
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [2, 3, 4, 5],
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', size: 12000, vhds: [`${relativePath}/child.vhd`] })
)
// Simulate a merge killed mid-transfer: write 2 of the 4 child blocks into the ancestor
// (which overwrites its end footer) then throw before mergeMetadata() rewrites the footer.
// This leaves the ancestor footerless AND a `.merge.json` state file — the exact post-crash
// state that makes cleanVm take the resume path.
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
await parent.init({ force: false })
await child.init({ force: false })
const merger = new MergeRemoteDisk(handler, { removeUnused: true, writeStateDelay: 0 })
let written = 0
const realWriteBlock = parent.writeBlock.bind(parent)
parent.writeBlock = async diskBlock => {
if (written === 2) {
throw new Error('simulated interruption')
}
written++
return realWriteBlock(diskBlock)
}
await assert.rejects(merger.merge(parent, child), /simulated interruption/)
assert.ok(
(await handler.list(basePath)).includes('.ancestor.vhd.merge.json'),
'an interrupted-merge state file should remain'
)
// Full pipeline: cleanVm must recognize the interrupted merge and resume it — opening the
// footerless ancestor through the chain without throwing "footer1 !== footer2".
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
merge: true,
logInfo: () => {},
logWarn: () => {},
})
// The merge completed: the ancestor was consolidated into child.vhd and the state file is gone.
const remaining = await handler.list(basePath)
assert.equal(remaining.includes('.ancestor.vhd.merge.json'), false, 'state file removed after successful resume')
assert.equal(remaining.includes('ancestor.vhd'), false, 'ancestor consolidated into the merge target')
assert.equal(remaining.includes('child.vhd'), true, 'merge target remains')
// No blocks lost: the resumed merge wrote the blocks (4, 5) that were never written before
// the crash, so the consolidated disk holds the full set.
const merged = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
await merged.init({ force: false })
try {
assert.deepEqual(merged.getBlockIndexes(), [0, 1, 2, 3, 4, 5], 'all blocks present after resumed merge')
} finally {
await merged.close()
}
// The backup is consistent, so its metadata is preserved.
assert.ok((await handler.list(rootPath)).includes('metadata.json'), 'metadata should be preserved')
})
test('it does not warn "missing target of alias" for aliases deleted by merge', async () => {
// Regression: RemoteDiskLineage.#cleanOrphanDataFiles iterated #diskPaths entries just
// deleted during the same clean() call's merge phase, producing false "missing target of
// alias" warnings
const orphan = await generateVhd(`${basePath}/orphan.vhd`, {
useAlias: true,
blocks: [0],
})
await generateVhd(`${basePath}/child.vhd`, {
useAlias: true,
header: {
parentUnicodeName: 'orphan.vhd.alias.vhd',
parentUuid: orphan.footer.uuid,
},
blocks: [1],
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd.alias.vhd`],
})
)
const warnings = []
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
merge: true,
logWarn: (message, data) => warnings.push({ message, data }),
logInfo: () => {},
})
const missingTargetWarnings = warnings.filter(w => w.message === 'missing target of alias')
assert.equal(
missingTargetWarnings.length,
0,
`"missing target of alias" must not fire after merge; got: ${JSON.stringify(missingTargetWarnings)}`
)
const remaining = await handler.list(basePath)
assert.ok(!remaining.includes('orphan.vhd.alias.vhd'), 'orphan alias must be deleted after merge')
assert.ok(remaining.includes('child.vhd.alias.vhd'), 'child alias must survive as merge target')
})
test('it resumes an interrupted merge with chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [5, 6],
})
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
blocks: [7, 8],
})
await handler.writeFile(
`${basePath}/.ancestor.vhd.merge.json`,
JSON.stringify({
parent: { uuid: '0' },
child: { uuid: '0' },
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
currentBlock: 3,
mergedDataSize: 3 * 2 * 1024 * 1024,
step: 'mergeBlocks',
diskSize: 0,
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const files = await handler.list(basePath)
assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
})
test('it resumes an interrupted merge without chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 209920,
vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
// a merge in progress file
await handler.writeFile(
`${basePath}/.orphan.vhd.merge.json`,
JSON.stringify({
parent: {
header: orphan.header.checksum,
},
child: {
header: child.header.checksum,
},
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 1)
assert.equal(remainingVhds.includes('child.vhd'), true)
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
describe('tests multiple combination ', { concurrency: 1 }, () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
// a broken VHD
if (useAlias) {
await handler.mkdir(basePath + '/data')
}
const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
if (vhdMode === 'directory') {
await handler.mkdir(brokenVhdDataPath)
} else {
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
}
if (useAlias) {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
await generateVhd(`${basePath}/abandoned.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// an ancestor of a vhd present in metadata
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
blocks: [1, 3],
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: ancestor.footer.uuid,
},
blocks: [1, 2],
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: child.footer.uuid,
},
blocks: [2, 3],
})
// an older parent that was merging in clean
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
useAlias,
mode: vhdMode,
})
// a clean vhd in metadata
const clean = await generateVhd(`${basePath}/clean.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: cleanAncestor.footer.uuid,
},
})
await handler.writeFile(
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
JSON.stringify({
parent: {
header: cleanAncestor.header.checksum,
},
child: {
header: clean.header.checksum,
},
})
)
// the metadata file
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
`${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
`${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
],
})
)
if (!useAlias && vhdMode === 'directory') {
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
} catch (err) {
assert.strictEqual(
err.code,
'NOT_SUPPORTED',
'Merging directory without alias should raise a not supported error'
)
return
}
assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 6501888)
// cache.json.gz must be regenerated when metadata changes after a merge
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after a merge'
)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
if (useAlias) {
const dataSurvivors = await handler.list(basePath + '/data')
// the goal of the alias : do not move a full folder
assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
assert.equal(survivors.length, 4) // the 3 ok + data
assert.equal(dataSurvivors.length, 3)
} else {
assert.equal(survivors.includes('clean.vhd'), true)
assert.equal(survivors.includes('child.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd'), true)
assert.equal(survivors.length, 3)
}
})
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
// metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('check all types of aliases, corrupted, missing or not', async () => {
// valid alias pointing to an existing data file
await handler.mkdir(`${basePath}/data`)
await generateVhd(`${basePath}/data/ok.vhd`)
await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
// broken alias pointing to a missing data file
await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
// alias pointing to an existing but corrupt data file
await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
// data file without alias
await generateVhd(`${basePath}/data/missingalias.vhd`)
// only the valid alias is referenced by a complete backup
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
// only ok.alias.vhd has survived
const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
assert.equal(aliases.length, 1)
assert.equal(aliases[0], 'ok.alias.vhd')
// corrupt.vhd and missingalias.vhd must be gone
const data = await handler.list(`${basePath}/data`)
assert.equal(data.length, 1)
assert.equal(data[0], 'ok.vhd')
})
test('it preserves VDI directory when handler.list() throws during lineage init', async () => {
// Regression: when handler.list(vdiDir) threw (e.g. S3 404), the lineage
// was never added to #uniqueLineages, so cleanOrphanDiskDirs treated the directory
// as an orphan and deleted it with rmtree()
await generateVhd(`${basePath}/snapshot.vhd`)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot.vhd`] })
)
const originalList = handler.list.bind(handler)
handler.list = async (path, opts) => {
if (typeof path === 'string' && path.includes(vdiId)) {
throw Object.assign(new Error('Simulated transient storage error'), { code: 'ENOENT' })
}
return originalList(path, opts)
}
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
} finally {
handler.list = originalList
}
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot.vhd'), 'VHD must survive when handler.list() throws during lineage init')
})
test('it regenerates the cache when its entry count is out of sync, even with nothing to merge/remove', async () => {
// a referenced disk + its metadata, but no cache.json.gz on disk
await generateVhd(`${basePath}/diskA1.vhd`)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`] })
)
await assert.rejects(handler.readFile(`${rootPath}/cache.json.gz`), { code: 'ENOENT' })
const logged = []
// remove/merge both false: cacheNeedsRegen stays false and no merge happens,
// so only the cache-count mismatch can trigger the regeneration
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: false,
merge: false,
logInfo: () => {},
logWarn: message => logged.push(message),
})
assert.ok(logged.includes('unexpected number of entries in backup cache'), 'mismatch should be reported')
const cache = JSON.parse((await gunzip(await handler.readFile(`${rootPath}/cache.json.gz`))).toString())
assert.equal(Object.keys(cache).length, 1, 'cache should be regenerated to match the archives on disk')
})

View File

@@ -1,174 +0,0 @@
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
ResolvedBackupCleanOptions,
VmBackupInterface,
PartialBackupMetadata,
} from './VmBackup.types.mjs'
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
export class VmIncrementalBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
metadataPath: string
metadata: PartialBackupMetadata
readonly diskPaths: Array<string>
rootPath: string
opts: ResolvedBackupCleanOptions
diskLineages: Map<string, RemoteDiskLineage> = new Map()
#isChecked = false
#isComplete = false
get isChecked(): boolean {
return this.#isChecked
}
get isComplete(): boolean {
return this.#isComplete
}
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
diskPaths: Array<string>, // xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/<snapshotUuid>.alias.vhd (one per VDI)
opts: ResolvedBackupCleanOptions,
lineageRegistry?: Map<string, RemoteDiskLineage>
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.diskPaths = diskPaths.map(path => normalize(path))
this.opts = opts
if (lineageRegistry !== undefined) {
this.diskLineages = lineageRegistry
}
}
async init(): Promise<void> {
// Build one RemoteDiskLineage per VDI directory derived from diskPaths
for (const diskPath of this.diskPaths) {
const vdiDir = dirname(diskPath)
if (!this.diskLineages.has(vdiDir)) {
const lineage = new RemoteDiskLineage(this.handler, vdiDir, this.opts)
this.diskLineages.set(vdiDir, lineage)
try {
await lineage.init()
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn('failed to scan VDI directory', { vdiDir, error })
}
}
}
}
async check(): Promise<CheckResult> {
const missingDisks: string[] = []
const brokenDisks: string[] = []
if (this.diskPaths.length === 0) {
this.opts.logWarn('incremental backup has no disk paths', { metadataPath: this.metadataPath })
this.#isComplete = false
} else {
for (const diskPath of this.diskPaths) {
const lineage = this.diskLineages.get(dirname(diskPath))
if (lineage === undefined || !lineage.isPathAccessible(diskPath)) {
if (lineage?.isBroken(diskPath)) {
this.opts.logWarn('disk check error', { path: diskPath })
brokenDisks.push(diskPath)
} else {
this.opts.logWarn('disk file missing', { path: diskPath })
missingDisks.push(diskPath)
}
}
}
this.#isComplete = missingDisks.length === 0 && brokenDisks.length === 0
if (!this.#isComplete) {
this.opts.logWarn('incremental backup is incomplete', {
metadataPath: this.metadataPath,
missingDisks,
brokenDisks,
})
}
}
if (this.#isComplete) {
for (const diskPath of this.diskPaths) {
this.diskLineages.get(dirname(diskPath))?.addActiveDiskPath(diskPath)
}
}
this.#isChecked = true
return { isValid: this.#isComplete, missingDisks, brokenDisks }
}
/**
* Removes the metadata file if the backup is incomplete (missing disks).
* If mergedSizes is provided, updates metadata with the total merged size for this archive's disks.
* Actual disk merge/deletion is handled by RemoteDiskLineage.
*/
async clean({ remove = this.opts.remove ?? false, mergedSizes }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (!this.#isChecked) {
await this.check()
}
const removedFiles: string[] = []
if (!this.#isComplete) {
removedFiles.push(this.metadataPath)
}
if (remove) {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
let mergedSize = 0
if (mergedSizes !== undefined) {
for (const diskPath of this.diskPaths) {
mergedSize += mergedSizes.get(diskPath) ?? 0
}
if (mergedSize > 0) {
await this.updateMetadata(mergedSize)
}
}
return { removedFiles }
}
/**
* Updates metadata.size after a merge and writes the updated metadata file to disk.
* Also resets isVhdDifferencing so all VDIs are marked as full (non-differencing).
*/
async updateMetadata(mergedSize: number): Promise<void> {
this.metadata.size = mergedSize
if (this.metadata.vdis !== undefined) {
this.metadata.isVhdDifferencing = {}
for (const id of Object.keys(this.metadata.vdis)) {
this.metadata.isVhdDifferencing[id] = false
}
}
await this.handler.writeFile(this.metadataPath, JSON.stringify(this.metadata), { flags: 'w' })
}
/**
* Returns the metadata file path and active disk paths if the backup is complete, empty otherwise.
* Must be called after check().
*/
getAssociatedFiles({ prefix = false }): Array<string> {
if (!this.#isComplete) {
return []
}
const files = [this.metadataPath, ...this.diskPaths]
return prefix ? files : files.map(file => basename(file))
}
}

View File

@@ -1,365 +0,0 @@
// @ts-check
/**
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
*/
import assert from 'assert'
import { createLogger } from '@xen-orchestra/log'
import { basename, dirname } from 'path'
import { asyncEach } from '@vates/async-each'
import { relativeFromFile, normalize, resolveFromFile } from '@xen-orchestra/fs/path'
// @ts-ignore
const { warn } = createLogger('remote-disk:merge')
/**
* @typedef {Object} MergeState
* @property {{ uuid?: string }} child
* @property {{ uuid?: string }} parent
* @property { string[] | undefined} chain
* @property {number} currentBlock
* @property {number} mergedDataSize
* @property {'mergeBlocks' | 'cleanup'} step
* @property {number} diskSize
* @typedef {(message: string, data?: Record<string, unknown>) => void} Logger
*/
export class MergeRemoteDisk {
/**
* @type {MergeState}
*/
#state = {
child: { uuid: '0' },
parent: { uuid: '0' },
chain: undefined,
currentBlock: 0,
mergedDataSize: 0,
step: 'mergeBlocks',
diskSize: 0,
}
/**
* @type {string}
*/
#statePath = ''
/**
* @type {boolean}
*/
#isResuming
/**
* @type {Logger | Function}
*/
#logInfo
/**
* @type {number}
*/
#mergeBlockConcurrency
/**
* @type {Function}
*/
#onProgress
/**
* @type {boolean}
*/
#removeUnused
/**
* @type {number}
*/
#writeStateDelay
/**
* @type {number}
*/
#lastStateWrittenAt
/**
* @type {RemoteHandlerAbstract}
*/
#handler
/**
* @param {RemoteHandlerAbstract} handler
* @param {Object} params
* @param {Function} [params.onProgress]
* @param {Logger | Function} [params.logInfo]
* @param {boolean} [params.removeUnused]
* @param {number} [params.mergeBlockConcurrency]
* @param {number} [params.writeStateDelay]
*/
constructor(
handler,
{
onProgress = () => {},
logInfo = () => {},
removeUnused = false,
mergeBlockConcurrency = 2,
writeStateDelay = 10e3,
}
) {
this.#handler = handler
this.#logInfo = logInfo
this.#onProgress = onProgress
this.#removeUnused = removeUnused
this.#mergeBlockConcurrency = mergeBlockConcurrency
this.#writeStateDelay = writeStateDelay
this.#isResuming = false
this.#lastStateWrittenAt = 0
}
/**
* @param {RemoteDisk} parentDisk
* @returns {Promise<boolean>} isResuming
*/
async isResuming(parentDisk) {
try {
await this.#handler.readFile(
dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
)
return true
} catch (error) {
return false
}
}
/**
* @param {RemoteDisk} parentDisk
* @param {RemoteDisk} childDisk
*/
async merge(parentDisk, childDisk) {
this.#statePath = dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
try {
const mergeStateContent = await this.#handler.readFile(this.#statePath)
this.#state = JSON.parse(mergeStateContent)
// work-around to bugs introduced in 97d94b795 and 4e50858
//
// currentBlock could be `null` due to the JSON.stringify of a `NaN` value
// could also be -1 due to being set as Math.min(...merging) - 1 if merging contains 0
if (this.#state?.currentBlock === null || this.#state?.currentBlock < 0) this.#state.currentBlock = 0
this.#isResuming = true
} catch (error) {
// @ts-ignore
if (error.code !== 'ENOENT') {
warn('problem while checking the merge state', { error })
}
}
try {
/* eslint-disable no-fallthrough */
switch (this.#state?.step ?? 'mergeBlocks') {
case 'mergeBlocks':
await this.#step_mergeBlocks(parentDisk, childDisk)
case 'cleanup':
await this.#step_cleanup(parentDisk, childDisk)
return this.#cleanup(parentDisk, childDisk, true)
default:
warn(`Step ${this.#state?.step} is unknown`, { state: this.#state })
}
/* eslint-enable no-fallthrough */
} catch (error) {
await this.#cleanup(parentDisk, childDisk, false)
throw error
}
}
async #writeState() {
try {
await this.#handler.writeFile(this.#statePath, JSON.stringify(this.#state), { flags: 'w' })
} catch (err) {
warn('failed to write merge state', { error: err })
}
}
async #writeStateThrottled() {
const now = Date.now()
if (now - this.#lastStateWrittenAt > this.#writeStateDelay) {
this.#lastStateWrittenAt = now
await this.#writeState()
}
}
/**
* @param {RemoteDisk} parentDisk
* @param {RemoteDisk} childDisk
*/
async #step_mergeBlocks(parentDisk, childDisk) {
const getMaxBlockCount = childDisk.getMaxBlockCount()
await parentDisk.resize(getMaxBlockCount)
if (this.#isResuming) {
const alreadyMergedBlocks = []
for (let blockId = 0; blockId < this.#state.currentBlock; blockId++) {
if (childDisk.hasBlock(blockId)) {
alreadyMergedBlocks.push(blockId)
}
}
parentDisk.setAllocatedBlocks(alreadyMergedBlocks)
} else {
this.#state.child = { uuid: childDisk.getUuid() ?? undefined }
this.#state.parent = { uuid: parentDisk.getUuid() ?? undefined }
this.#state.chain = [parentDisk.getPath(), ...childDisk.getPaths()].map(path =>
relativeFromFile(this.#statePath, path)
)
// Finds first allocated block for the 2 following loops
while (this.#state.currentBlock < getMaxBlockCount && !childDisk.hasBlock(this.#state.currentBlock)) {
++this.#state.currentBlock
}
await this.#writeState()
}
await this.#mergeBlocks(parentDisk, childDisk)
await parentDisk.flushMetadata(childDisk)
await parentDisk.mergeMetadata(childDisk)
this.#state.diskSize = parentDisk.getSizeOnDisk()
}
/**
* @param {RemoteDisk} parentDisk
* @param {RemoteDisk} childDisk
*/
async #mergeBlocks(parentDisk, childDisk) {
this.#mergeBlockConcurrency =
(await parentDisk.canMergeConcurently()) && (await childDisk.canMergeConcurently())
? this.#mergeBlockConcurrency
: 1
const toMerge = []
for (let block = this.#state.currentBlock; block < childDisk.getMaxBlockCount(); block++) {
if (childDisk.hasBlock(block)) {
toMerge.push(block)
}
}
const nBlocks = toMerge.length
this.#onProgress({ total: nBlocks, done: 0 })
const merging = new Set()
let counter = 0
await asyncEach(
toMerge,
async blockId => {
merging.add(blockId)
const blockSize = await parentDisk.mergeBlock(childDisk, blockId, this.#isResuming)
this.#state.mergedDataSize += blockSize
this.#state.currentBlock = Math.max(0, Math.min(...merging) - 1)
merging.delete(blockId)
this.#onProgress({ total: nBlocks, done: counter + 1 })
counter++
await this.#writeStateThrottled()
},
{ concurrency: this.#mergeBlockConcurrency }
)
await this.#writeState()
this.#onProgress({ total: nBlocks, done: nBlocks })
}
/**
* @param {RemoteDisk} parentDisk
* @param {RemoteDisk} childDisk
*/
async #step_cleanup(parentDisk, childDisk) {
assert.notEqual(this.#state, undefined)
this.#state.step = 'cleanup'
await this.#writeState()
const mergeTargetPath = childDisk.getPath()
// delete intermediate children if needed
if (this.#removeUnused) {
await childDisk.unlink()
}
try {
await parentDisk.rename(mergeTargetPath)
} catch (error) {
// @ts-ignore
if (error.code === 'ENOENT' && this.#isResuming) {
// @ts-ignore
this.#logInfo(`the parent disk was already renamed`, {
parent: parentDisk.getPath(),
mergeTarget: mergeTargetPath,
})
} else {
throw error
}
}
}
/**
* @param {RemoteDisk} parentDisk
* @param {RemoteDisk} childDisk
* @param {boolean} cleanStateFile
*
* @returns {Promise<{mergedDataSize: number, finalDiskSize: number}>} result
*/
async #cleanup(parentDisk, childDisk, cleanStateFile) {
const finalDiskSize = this.#state?.diskSize ?? 0
const mergedDataSize = this.#state?.mergedDataSize ?? 0
await parentDisk.close().catch(warn)
await childDisk.close().catch(warn)
if (cleanStateFile) {
await this.#handler.unlink(this.#statePath).catch(warn)
}
return { mergedDataSize, finalDiskSize }
}
/**
* Scans vdiDir files for interrupted merge state files and resolves their chains
*
* @param {RemoteHandlerAbstract} handler
* @param {string} vdiDir
* @param {string[]} files - all file paths in vdiDir (prependDir: true)
* @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
* @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
*/
static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
const result = new Map()
for (const filePath of files) {
const match = STATE_FILE_RE.exec(basename(filePath))
if (match === null) continue
const parentPath = normalize(vdiDir + '/' + match[1])
let chain
try {
const state = JSON.parse(await handler.readFile(filePath))
if (Array.isArray(state?.chain)) {
chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
} else {
// old state file without chain: fall back to UUID lookup then header-based map
const childUuid = state?.child?.uuid
const childPath = childUuid !== undefined ? uuidToPath.get(childUuid) : childOf.get(parentPath)
if (childPath !== undefined) chain = [parentPath, childPath]
}
} catch (error) {
warn("Merge state unreadable, can't restart merging.", error)
}
result.set(parentPath, { stateFilePath: filePath, chain })
}
return result
}
}

View File

@@ -1,282 +0,0 @@
// @ts-check
/**
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
*/
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
export class RemoteDisk extends RandomAccessDisk {
/**
* Abstract
* @param {Object} options
* @param {boolean} options.force
* @returns {Promise<void>}
*/
async init(options = { force: false }) {
throw new Error(`init must be implemented`)
}
/**
* Abstract
* @returns {Promise<void>}
*/
async close() {
throw new Error(`close must be implemented`)
}
/**
* Abstract
* @returns {number}
*/
getVirtualSize() {
throw new Error(`getVirtualSize must be implemented`)
}
/**
* Abstract
* @returns {number} size
*/
getSizeOnDisk() {
throw new Error(`getSizeOnDisk must be implemented`)
}
/**
* @returns {number}
*/
getBlockSize() {
throw new Error(`getBlockSize must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getPath() {
throw new Error(`getPath must be implemented`)
}
/**
* Abstract
* Returns an array of disk paths.
*
* @returns {string[]}
*/
getPaths() {
throw new Error(`getPaths must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getUuid() {
throw new Error(`getUuid must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getParentUuid() {
throw new Error(`getParentUuid must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getParentPath() {
throw new Error(`getParentPath must be implemented`)
}
/**
* Abstract
* @returns {object}
*/
getMetadata() {
throw new Error(`getMetadata must be implemented`)
}
/**
* Abstract
* @returns {Promise<boolean>} canMergeConcurently
*/
async canMergeConcurently() {
throw new Error(`canMergeConcurently must be implemented`)
}
/**
* @returns {number} getMaxBlockCount
*/
getMaxBlockCount() {
return Math.ceil(this.getVirtualSize() / this.getBlockSize())
}
/**
* Returns true if every block present in `other` is also present in this disk.
* @param {RemoteDisk} other
* @returns {boolean}
*/
containsAllDataOf(other) {
for (const blockIndex of other.getBlockIndexes()) {
if (!this.hasBlock(blockIndex)) return false
}
return true
}
/**
* Checks if the VHD contains a specific block.
* @param {number} index
* @returns {boolean}
*/
hasBlock(index) {
throw new Error(`hasBlock must be implemented`)
}
/**
* Abstract
* Gets the indexes of all blocks in the VHD.
* @returns {Array<number>}
*/
getBlockIndexes() {
throw new Error(`getBlockIndexes must be implemented`)
}
/**
* Abstract
* Returns the parent non inizialized instance
* @returns {RemoteDisk}
*/
instantiateParent() {
throw new Error(`instantiateParent must be implemented`)
}
/**
* Abstract
* Writes a full block.
* @param {DiskBlock} diskBlock
* @return {Promise<number>} blockSize
*/
async writeBlock(diskBlock) {
throw new Error(`writeBlock must be implemented`)
}
/**
* Abstract
* Reads a specific block from the VHD.
* @param {number} index
* @returns {Promise<DiskBlock>} diskBlock
*/
async readBlock(index) {
throw new Error(`readBlock must be implemented`)
}
/**
* Abstract
* Reads a specific block from the child disk to copy/move it to this disk.
* @param {RemoteDisk} childDisk
* @param {number} index
* @param {boolean} isResumingMerge
* @returns {Promise<number>} blockSize
*/
async mergeBlock(childDisk, index, isResumingMerge) {
throw new Error(`mergeBlock must be implemented`)
}
/**
* Abstract
* Manually set the disk allocated blocks.
* @param {Array<number>} blockIds
* @returns {Promise<void>}
*/
async setAllocatedBlocks(blockIds) {
throw new Error(`setAllocatedBlocks must be implemented`)
}
/**
* Abstract
* @param {number} blockCount
* @returns {Promise<void>}
*/
async resize(blockCount) {
throw new Error(`resize must be implemented`)
}
/**
* Abstract
* @param {RemoteDisk} childDisk
* @returns {Promise<void>}
*/
async flushMetadata(childDisk) {
throw new Error(`flushMetadata must be implemented`)
}
/**
* Abstract
* @param {RemoteDisk} childDisk
* @returns {Promise<void>}
*/
mergeMetadata(childDisk) {
throw new Error(`mergeMetadata must be implemented`)
}
/**
* Abstract
* Checks if the VHD is a differencing disk.
* @returns {boolean}
*/
isDifferencing() {
throw new Error(`isDifferencing must be implemented`)
}
/**
* Abstract
* Rename alias/disk
* @param {string} newPath
*/
async rename(newPath) {
throw new Error(`rename must be implemented`)
}
/**
* Abstract
* Deletes alias/disk/disks
* @param {Object} options
*/
async unlink({ force = false } = {}) {
throw new Error(`unlink must be implemented`)
}
/**
* Checks the integrity of the disk's external reference (e.g. alias target).
* @param {Object} [opts]
* @param {boolean} [opts.remove]
* @param {Function} [opts.logWarn]
* @param {Function} [opts.logInfo]
* @returns {Promise<string | undefined>}
*/
async clean(opts = {}) {
return undefined
}
/**
* Returns all file paths within dir that this disk claims.
*
* @param {string} dir
* @returns {Promise<string[]>}
*/
async listAssociatedFiles(dir) {
throw new Error(`listAssociatedFiles must be implemented`)
}
/**
* @returns {Promise<RemoteDisk>}
*/
async openParent() {
const parent = await super.openParent()
if (!(parent instanceof RemoteDisk)) {
throw new Error('parent of a RemoteDisk must be also a RemoteDisk')
}
return parent
}
}

Some files were not shown because too many files have changed in this diff Show More