Compare commits

..

4 Commits

Author SHA1 Message Date
Florent BEAUCHAMP
6d311d9b11 wip 2026-01-12 17:00:48 +00:00
Florent BEAUCHAMP
06547f0a6f add typing 2026-01-08 17:01:08 +00:00
Florent BEAUCHAMP
e1b7077dc1 skip test that can't work on encrypted remote 2026-01-08 12:37:01 +00:00
Florent BEAUCHAMP
790201ef8b feat(fs): add rolling encryption
This PR add the ability to update the encrryption key of a remote / backup repository

To acheive this goal, a few changes have been mades :

 * The files will be prepended with a the index of key (unencrypted)
 * The encrypted part of the file will also contains this index, to ensure it
is not toyed with, and authenticated by the encryption algorithm
 * The latest encryption key is still not stored in the BR
 * the previous keys are stored in encryptors/{index], each key encrypted by the
following one

This means that you can decrypt all the keys with only the latest one
Another consequence is that the oldest incremental backup can contains a mix of
files encrytped with different keys, and it's ok since each file

This can onoy be enabled on a a new BR, since the files must have a header indicating
the key used

Oleder key are not meant to be deleted, but if mandatory, the user can delete the keys
that are older than the most recent
2026-01-08 10:26:53 +00:00
2658 changed files with 37824 additions and 143309 deletions

View File

@@ -85,21 +85,6 @@ module.exports = {
XO_LITE_GIT_HEAD: true,
},
rules: {
'import/extensions': [
'error',
'ignorePackages',
{
pattern: { ts: 'always', vue: 'always' },
checkTypeImports: true,
// '@core/*' and '@/*' look like scoped packages to the plugin, which 'ignorePackages' would skip
pathGroupOverrides: [
{ pattern: '@core/packages/*', action: 'ignore' }, // barrel files
{ pattern: '@core/icons', action: 'ignore' },
{ pattern: '@core/**', action: 'enforce' },
{ pattern: '@/**', action: 'enforce' },
],
},
],
'import/order': [
'error',
{
@@ -248,30 +233,7 @@ module.exports = {
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
'@intlify/vue-i18n/no-missing-keys-in-other-locales': [
'error',
{
ignoreLocales: [
'de',
'fa',
'cs',
'es',
'sv',
'it',
'ru',
'uk',
'nl',
'pt-BR',
'da',
'ko',
'nb-NO',
'ja',
'fi',
'pl',
'zh-Hans',
'sk',
'pt',
'ar',
],
},
{ ignoreLocales: ['de', 'fa', 'cs', 'es', 'sv', 'it', 'ru', 'uk', 'nl', 'pt-BR', 'da', 'ko', 'nb-NO', 'ja'] },
],
},
},
@@ -297,7 +259,7 @@ module.exports = {
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/src/route-map.d.ts'],
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
parserOptions: {
sourceType: 'module',
},
@@ -306,17 +268,6 @@ module.exports = {
'eslint-comments/no-unlimited-disable': 'off',
},
},
{
files: ['*.d.ts'],
parser: '@typescript-eslint/parser',
parserOptions: {
sourceType: 'module',
},
rules: {
'n/no-unsupported-features/es-syntax': 'off',
'no-unused-vars': 'off',
},
},
],
ignorePatterns: ['@vates/fatfs/'],

View File

@@ -25,7 +25,7 @@ jobs:
- name: Setup Node environment
uses: actions/setup-node@v4
with:
node-version: '>=22.12'
node-version: '20'
cache: 'yarn'
- name: Install project dependencies

View File

@@ -3,9 +3,4 @@
**/*.hbs
# Generated files
/@xen-orchestra/web/src/route-map.d.ts
/@xen-orchestra/lite/src/route-map.d.ts
# Docusaurus MDX content: prettier's markdown parser corrupts JSX
# comments ({/* … */} becomes {/_ … _/}) and other MDX constructs
/docs/docs/**/*.md
/@xen-orchestra/web/typed-router.d.ts

View File

@@ -1,5 +0,0 @@
export function asyncEach<Item>(
iterable: Iterable<Item> | AsyncIterable<Item>,
iteratee: (item: Item, index: number, iterable: Iterable<Item> | AsyncIterable<Item>) => Promise<void>,
opts?: { concurrency?: number; signal?: AbortSignal; stopOnError?: boolean }
): Promise<void>

View File

@@ -67,17 +67,20 @@ describe('asyncEach', () => {
})
it('stops on first error when stopOnError is true', async () => {
const tracker = new assert.CallTracker()
const error = new Error()
const iteratee = spy((_, i) => {
const iteratee = tracker.calls((_, i) => {
if (i === 1) {
throw error
}
})
}, 2)
assert.deepStrictEqual(
await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true })),
error
)
assert.deepStrictEqual(iteratee.callCount, 2)
tracker.verify()
})
it('rejects AggregateError when stopOnError is false', async () => {
@@ -97,16 +100,19 @@ describe('asyncEach', () => {
})
it('can be interrupted with an AbortSignal', async () => {
const tracker = new assert.CallTracker()
const ac = new AbortController()
const iteratee = spy((_, i) => {
const iteratee = tracker.calls((_, i) => {
if (i === 1) {
ac.abort()
}
})
}, 2)
await assert.rejects(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal }), {
message: 'asyncEach aborted',
})
assert.deepStrictEqual(iteratee.callCount, 2)
tracker.verify()
})
it('handle error on iterable', async () => {

View File

@@ -24,8 +24,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"types": "index.d.ts",
"version": "1.0.3",
"version": "1.0.1",
"engines": {
"node": ">=8.10"
},

View File

@@ -24,8 +24,8 @@
},
"dependencies": {
"@vates/multi-key-map": "^0.2.0",
"@xen-orchestra/async-map": "^0.1.4",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.7.1",
"ensure-array": "^1.0.0"
},
"devDependencies": {

View File

@@ -1,12 +1,12 @@
import Fuse from 'fuse-native'
import LRU from 'lru-cache'
import Fuse from 'fuse-native'
import { createLogger } from '@xen-orchestra/log'
import { VhdSynthetic } from 'vhd-lib'
import { Disposable, fromCallback } from 'promise-toolbox'
const { warn } = createLogger('vates:fuse-vhd')
// build a stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
const stat = st => ({
mtime: st.mtime || new Date(),
atime: st.atime || new Date(),
@@ -20,109 +20,42 @@ const stat = st => ({
export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
const cache = new LRU({
max: 16, // each cached block is 2MB in size
})
await vhd.readBlockAllocationTable()
const { blockSize } = vhd.header
// A zeroed buffer returned for VHD blocks that are not present in the chain.
// Never mutated — safe to share across all reads.
const EMPTY_BLOCK = Buffer.alloc(blockSize, 0)
// --- coalescing block read queue ---
//
// FUSE read requests from NTFS-3g arrive in small chunks (typically 4 KiB)
// while VHD blocks are 2 MiB. Reading blocks one at a time from the backing
// store (CIFS/NFS/…) bounds libuv threadpool usage to 1 concurrent I/O call.
//
// Coalescing: all pending requests for the same blockId share a single
// readBlock() call. When the block resolves, every waiter is fulfilled at once.
//
// Together these two properties prevent the FUSE-on-FUSE threadpool deadlock:
// concurrent NTFS-3g reads no longer starve the VHD block reads they depend on.
// In-memory block cache. Each VHD block is 2 MiB; 16 blocks = 32 MiB.
// This is the primary tool against the 300× CIFS amplification: NTFS-3g
// reads a 2 MiB block in ~512 sequential 4 KiB FUSE requests. Without a
// cache, each request causes a full 2 MiB CIFS read. With the cache, only
// the first request reads from CIFS; the rest are served from RAM.
const blockCache = new LRU({ max: 64 })
const pendingBlocks = new Map() // blockId → Promise<Buffer>
const readQueue = [] // { blockId, resolve, reject }[]
let queueRunning = false
// log stats every 10 s while there is activity
function enqueueBlock(blockId) {
// 1. serve from cache if available
if (blockCache.has(blockId)) {
return Promise.resolve(blockCache.get(blockId))
}
// 2. coalesce: join an existing in-flight read for this block
if (pendingBlocks.has(blockId)) {
return pendingBlocks.get(blockId)
}
const p = new Promise((resolve, reject) => {
readQueue.push({ blockId, resolve, reject })
})
pendingBlocks.set(blockId, p)
p.then(
data => {
pendingBlocks.delete(blockId)
blockCache.set(blockId, data)
},
() => pendingBlocks.delete(blockId)
)
drainQueue()
return p
}
function drainQueue() {
if (queueRunning) return
queueRunning = true
// async IIFE — errors are forwarded to individual reject() callbacks,
// so the outer floating promise is always fulfilled
;(async () => {
while (readQueue.length > 0) {
const { blockId, resolve, reject } = readQueue.shift()
try {
resolve((await vhd.readBlock(blockId)).data)
} catch (err) {
reject(err)
}
}
queueRunning = false
})()
}
async function readData(buf, len, pos) {
if (len === 0) return 0
const startBlockId = Math.floor(pos / blockSize)
// use (pos + len - 1) so an exact block boundary doesn't include the next block
const endBlockId = Math.floor((pos + len - 1) / blockSize)
let copied = 0
for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
const data = vhd.containsBlock(blockId) ? await enqueueBlock(blockId) : EMPTY_BLOCK
const offsetStart = blockId === startBlockId ? pos % blockSize : 0
const offsetEnd = blockId === endBlockId ? ((pos + len - 1) % blockSize) + 1 : blockSize
data.copy(buf, copied, offsetStart, offsetEnd)
copied += offsetEnd - offsetStart
}
return copied
}
const fuse = new Fuse(mountDir, {
async readdir(path, cb) {
if (path === '/') return cb(null, ['vhd0'])
if (path === '/') {
return cb(null, ['vhd0'])
}
cb(Fuse.ENOENT)
},
async getattr(path, cb) {
if (path === '/') return cb(null, stat({ mode: 'dir', size: 4096 }))
if (path === '/vhd0') return cb(null, stat({ mode: 'file', size: vhd.footer.currentSize }))
if (path === '/') {
return cb(
null,
stat({
mode: 'dir',
size: 4096,
})
)
}
if (path === '/vhd0') {
return cb(
null,
stat({
mode: 'file',
size: vhd.footer.currentSize,
})
)
}
cb(Fuse.ENOENT)
},
read(path, fd, buf, len, pos, cb) {
if (path === '/vhd0') {
return readData(buf, len, pos).then(cb, error => {
return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
warn('read error', { path, len, pos, error })
cb(Fuse.EIO)
})
@@ -131,9 +64,7 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
},
})
return new Disposable(
() => {
return fromCallback(cb => fuse.unmount(cb))
},
() => fromCallback(cb => fuse.unmount(cb)),
fromCallback(cb => fuse.mount(cb))
)
})

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/fuse-vhd",
"version": "2.1.3",
"version": "2.1.2",
"license": "ISC",
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,12 +19,12 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.9.3",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/fs": "^4.6.5",
"@xen-orchestra/log": "^0.7.1",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.16.1"
"vhd-lib": "^4.14.6"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/generator-toolbox",
"version": "1.1.3",
"version": "1.1.0",
"main": "dist/index.mjs",
"license": "MIT",
"private": false,
@@ -10,7 +10,7 @@
"rimraf": "^6.0.1",
"@types/node": "^20.6",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0"
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",

View File

@@ -144,7 +144,14 @@ class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
return this.#parent.remove(this.#uid, e)
}
[Symbol.asyncIterator](): AsyncGenerator<T, TReturn, TNext> {
return this
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
while (true) {
const res = await this.next()
if (res.done) {
break
}
yield res.value
}
}
}

View File

@@ -2,7 +2,8 @@ import assert from 'node:assert'
import { suite, test } from 'node:test'
import { Synchronized } from './synchronized.mjs'
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (_val: unknown) => {}) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (val: unknown) => {}) {
for (let i = 0; i < end; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
@@ -16,7 +17,7 @@ async function consume(
iterable: AsyncGenerator,
delay = 2500,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
onConsumed = (_val: unknown, _iterable: AsyncGenerator) => Promise.resolve(false)
onConsumed = (val: unknown, iterable: AsyncGenerator) => Promise.resolve(false)
) {
for await (const val of iterable) {
await new Promise(resolve => setTimeout(resolve, delay))
@@ -27,30 +28,6 @@ async function consume(
}
suite('success', () => {
test('for-await-of break stops the source generator', async () => {
let sourceClosed = false
const source = (async function* () {
try {
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
}
} finally {
sourceClosed = true
}
})()
const forker = new Synchronized(source)
const fork = forker.fork('first')
// for-await-of must call fork.return() on break, which propagates to the source
for await (const val of fork) {
if (val === 2) break
}
assert.strictEqual(sourceClosed, true)
})
test('if works with multiple consumer', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(3, progress)

View File

@@ -1,7 +1,4 @@
import assert from 'node:assert'
const TIMEOUT = Symbol('timeout')
export class Timeout<T> implements AsyncGenerator {
#source: AsyncGenerator<T>
#timeout: number
@@ -11,30 +8,21 @@ export class Timeout<T> implements AsyncGenerator {
this.#timeout = timeout
}
async next(): Promise<IteratorResult<T>> {
let timeoutHandle: ReturnType<typeof setTimeout>
const sourceNext = this.#source.next()
const result = await Promise.race([
sourceNext.then(res => {
clearTimeout(timeoutHandle)
return res
}),
new Promise<typeof TIMEOUT>(resolve => {
timeoutHandle = setTimeout(() => resolve(TIMEOUT), this.#timeout)
}),
])
if (result === TIMEOUT) {
// stop the source once the in-flight next settles to avoid data loss on the next call
// we don't want to await for the end of timeouted code
sourceNext.then(
() => this.#source.return(undefined).catch(() => {}),
() => {} // source already errored, nothing to clean up
)
throw new Error('Timeout reached')
}
return result
let timeout: ReturnType<typeof setTimeout>
const promiseTimeout = new Promise((_, reject) => {
timeout = setTimeout(() => {
reject(new Error('Timeout reached '))
}, this.#timeout)
})
const promiseNext = new Promise((resolve, reject) => {
this.#source.next().then(res => {
// ensure timetout won't fire later
clearTimeout(timeout)
resolve(res)
}, reject)
})
// promiseTimeout will never resolve
return Promise.race([promiseNext, promiseTimeout]) as Promise<IteratorResult<T>>
}
return(): Promise<IteratorResult<T>> {
return this.#source.return(undefined)

View File

@@ -64,24 +64,4 @@ describe('Timeout class', () => {
await assert.rejects(timeout.throw(new Error('Test error')), 'Expected error was not thrown')
})
it('should close the source generator after a timeout', async () => {
let sourceClosed = false
const slowSource = (async function* () {
try {
// resolves after 100ms so the in-flight next() eventually settles
await new Promise(resolve => setTimeout(resolve, 100))
yield 1
} finally {
sourceClosed = true
}
})()
const timeout = new Timeout(slowSource, 10) // fires before source yields
await assert.rejects(() => timeout.next(), /Timeout reached/)
// wait for the in-flight next() to settle so source.return() can be called
await new Promise(resolve => setTimeout(resolve, 200))
assert.strictEqual(sourceClosed, true)
})
})

View File

@@ -1,65 +0,0 @@
### `create([opts], [requestListener])`
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
```js
import { create } from '@vates/http-server-plus'
const server = create(function onRequest(req, res) {
res.end('Hello')
})
```
### `server.listen(opts)`
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
```js
// HTTP on a specific port
await server.listen({ port: 8080 })
// HTTPS — detected automatically when cert+key are provided
await server.listen({
port: 443,
cert: fs.readFileSync('certificate.pem'),
key: fs.readFileSync('private_key.pem'),
})
// Unix socket
await server.listen({ socket: '/var/run/http.sock' })
// File descriptor
await server.listen({ fd: 3 })
// Systemd socket activation
await server.listen({ systemdSocket: 0 })
```
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
Multiple endpoints can be started on the same instance:
```js
const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
console.log('listening on', httpAddr, httpsAddr)
```
### HTTP/2
Pass a custom `createSecureServer` factory to enable HTTP/2:
```js
import { createSecureServer } from 'node:http2'
import { create } from '@vates/http-server-plus'
const server = create({ createSecureServer }, app)
await server.listen({ port: 443, cert, key })
```
### `server.close()`
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
```js
await server.close()
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,98 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/http-server-plus
[![Package Version](https://badgen.net/npm/v/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus) ![License](https://badgen.net/npm/license/@vates/http-server-plus) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/http-server-plus)](https://bundlephobia.com/result?p=@vates/http-server-plus) [![Node compatibility](https://badgen.net/npm/node/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus)
> Augmented &#x60;http.Server&#x60;, HTTP/HTTPS/HTTP2 and multiple ports on the same instance
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/http-server-plus):
```sh
npm install --save @vates/http-server-plus
```
## Usage
### `create([opts], [requestListener])`
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
```js
import { create } from '@vates/http-server-plus'
const server = create(function onRequest(req, res) {
res.end('Hello')
})
```
### `server.listen(opts)`
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
```js
// HTTP on a specific port
await server.listen({ port: 8080 })
// HTTPS — detected automatically when cert+key are provided
await server.listen({
port: 443,
cert: fs.readFileSync('certificate.pem'),
key: fs.readFileSync('private_key.pem'),
})
// Unix socket
await server.listen({ socket: '/var/run/http.sock' })
// File descriptor
await server.listen({ fd: 3 })
// Systemd socket activation
await server.listen({ systemdSocket: 0 })
```
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
Multiple endpoints can be started on the same instance:
```js
const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
console.log('listening on', httpAddr, httpsAddr)
```
### HTTP/2
Pass a custom `createSecureServer` factory to enable HTTP/2:
```js
import { createSecureServer } from 'node:http2'
import { create } from '@vates/http-server-plus'
const server = create({ createSecureServer }, app)
await server.listen({ port: 443, cert, key })
```
### `server.close()`
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
```js
await server.close()
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,268 +0,0 @@
'use strict'
// ===================================================================
const EventEmitter = require('events').EventEmitter
const formatUrl = require('url').format
const fromEvent = require('promise-toolbox/fromEvent')
const inherits = require('util').inherits
const resolvePath = require('path').resolve
// ===================================================================
const forwardedEvents = [
'checkContinue',
'checkExpectation',
'clientError',
'connect',
'connection',
'listening',
'request',
'secureConnection',
'upgrade',
]
function extractProperty(obj, prop) {
const value = obj[prop]
if (value !== undefined) {
delete obj[prop]
return value
}
}
function forOwn(obj, cb) {
const keys = Object.keys(obj)
for (let i = 0, n = keys.length; i < n; ++i) {
cb(obj[keys[i]])
}
}
function getSystemdFd(index) {
if (process.pid !== +process.env.LISTEN_PID) {
throw new Error('systemd sockets are not meant for us (LISTEN_PID)')
}
const listenFds = process.env.LISTEN_FDS
if (listenFds == null) {
throw new Error('no systemd sockets found (LISTEN_FDS)')
}
const max = listenFds - 1
if (index > max) {
throw new Error('no such systemd socket found')
}
return index + 3 // 3 = SD_LISTEN_FDS_START
}
const isEmpty = obj => Object.keys(obj).length === 0
// ===================================================================
function onNewListener(event, listener) {
if (forwardedEvents.includes(event)) {
forOwn(this._servers, server => {
server.on(event, listener)
})
}
}
function onRemoveListener(event, listener) {
if (forwardedEvents.includes(event)) {
forOwn(this._servers, server => {
server.off(event, listener)
})
}
}
function Server(opts) {
EventEmitter.call(this)
this._createServer = (opts != null && opts.createServer) || require('http').createServer
this._createSecureServer = (opts != null && opts.createSecureServer) || require('https').createServer
this._servers = Object.create(null)
this._niceAddresses = Object.create(null)
this.on('removeListener', onRemoveListener).on('newListener', onNewListener)
}
inherits(Server, EventEmitter)
const proto = Server.prototype
function getAddress(serverId) {
return this[serverId].address()
}
proto.addresses = function Server$addresses() {
const servers = this._servers
return Object.keys(servers).map(getAddress, servers)
}
function getNiceAddress(serverId) {
return this[serverId]
}
proto.niceAddresses = function Server$niceAddresses() {
const niceAddresses = this._niceAddresses
return Object.keys(niceAddresses).map(getNiceAddress, niceAddresses)
}
function close(server) {
server.close()
}
proto.close = function Server$close(callback) {
if (callback) {
this.once('close', callback)
}
// Emit the close event even if there are no registered servers.
if (isEmpty(this._servers)) {
const self = this
setImmediate(function () {
self.emit('close')
})
return Promise.resolve()
}
// Closes each servers.
forOwn(this._servers, close)
return fromEvent(this, 'close')
}
let nextId = 0
proto.listen = function Server$listen(opts) {
opts = Object.assign({}, opts)
if (opts.hostname === 'localhost') {
return Promise.all([
this.listen(Object.assign(opts, { hostname: '::1' })),
this.listen(Object.assign(opts, { hostname: '127.0.0.1' })),
]).then(addresses => addresses[0])
}
const hostname = extractProperty(opts, 'hostname')
let port = extractProperty(opts, 'port')
const systemdSocket = extractProperty(opts, 'systemdSocket')
let fd = extractProperty(opts, 'fd')
let socket = extractProperty(opts, 'socket')
const servers = this._servers
const niceAddresses = this._niceAddresses
let server, protocol
if (opts.pfx || opts.SNICallback || (opts.cert && opts.key)) {
server = this._createSecureServer(opts)
protocol = 'https'
} else {
server = this._createServer(opts)
protocol = 'http'
}
const id = nextId++
servers[id] = server
if (systemdSocket != null) {
fd = getSystemdFd(systemdSocket)
}
// Compute a temporary nice address to display in case of error.
let niceAddress
if (fd != null) {
server.listen({ fd })
niceAddress = protocol + '://<fd:' + fd + '>'
} else if (socket != null) {
socket = resolvePath(socket)
server.listen(socket)
niceAddress = protocol + '://' + socket
} else {
if (port == null) {
port = protocol === 'https' ? 443 : 80
}
server.listen(port, hostname)
niceAddress = formatUrl({
protocol,
// Hostname default to localhost.
hostname: hostname || 'localhost',
// No port means random, unknown for now.
port: port || '<unknown>',
})
}
const emit = this.emit.bind(this)
server.once('close', function onClose() {
delete servers[id]
delete niceAddresses[id]
if (isEmpty(servers)) {
emit('close')
}
})
server.on('error', function onError() {
delete servers[id]
// FIXME: Should it be forwarded and be fatal if there are no
// listeners?
})
forwardedEvents.forEach(event => {
this.listeners(event).forEach(listener => {
server.on(event, listener)
})
})
return fromEvent(server, 'listening').then(
function () {
const address = server.address()
const niceAddress =
typeof address === 'string'
? protocol + '://' + address
: formatUrl({
protocol,
hostname: address.address,
port: address.port,
})
niceAddresses[id] = niceAddress
return niceAddress
},
function (error) {
error.niceAddress = niceAddress
throw error
}
)
}
function ref(server) {
server.ref()
}
proto.ref = function Server$ref() {
forOwn(this._servers, ref)
}
function unref(server) {
server.unref()
}
proto.unref = function Server$unref() {
forOwn(this._servers, unref)
}
// ===================================================================
module.exports = exports = Server
exports.create = function create(opts, requestListener) {
if (typeof opts === 'function') {
requestListener = opts
opts = undefined
}
const server = new Server(opts)
if (requestListener !== undefined) {
server.on('request', requestListener)
}
return server
}

View File

@@ -1,303 +0,0 @@
'use strict'
// ===================================================================
const assert = require('assert/strict')
const { EventEmitter } = require('events')
const resolvePath = require('path').resolve
const { describe, it } = require('node:test')
const HttpServerPlus = require('./')
// ===================================================================
// Creates a minimal mock server that can be used as a drop-in for http/https.Server.
// `address` is the value returned by mock.address() — use an object for TCP, a string for sockets.
function createMock(address) {
const mock = new EventEmitter()
mock.listen = function () {
setImmediate(() => this.emit('listening'))
}
mock.address = () => address
mock.close = function () {
setImmediate(() => this.emit('close'))
}
return mock
}
// ===================================================================
describe('HttpServerPlus', function () {
describe('#create()', function () {
it('creates a new instance', function () {
assert(HttpServerPlus.create() instanceof HttpServerPlus)
})
it('can register a `request` listener', function () {
const listener = function () {}
const server = HttpServerPlus.create(listener)
assert.deepEqual(server.listeners('request'), [listener])
})
})
describe('.addresses()', function () {
it('returns the list of addresses the server is listening at', function () {
const addr = { address: '127.0.0.1', port: 8080, family: 'IPv4' }
const mock = createMock(addr)
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(function () {
assert.deepEqual(server.addresses(), [addr])
return server.close()
})
})
})
describe('.niceAddresses()', function () {
it('returns the list of human readable addresses the server is listening at', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function (address) {
assert.deepEqual(server.niceAddresses(), [address])
return server.close()
})
})
})
describe('secureConnection event forwarding', function () {
function createMockHttpsServer() {
const server = new EventEmitter()
server.listen = function () {
setImmediate(() => this.emit('listening'))
}
server.address = () => ({ address: '127.0.0.1', port: 8443 })
server.close = function () {
setImmediate(() => this.emit('close'))
}
return server
}
function createServerWithMock() {
let mockServer
const server = HttpServerPlus.create({
createSecureServer: () => {
mockServer = createMockHttpsServer()
return mockServer
},
})
return {
server,
getMock: () => mockServer,
}
}
it('forwards secureConnection to listeners registered before listen()', function () {
const { server, getMock } = createServerWithMock()
const received = []
server.on('secureConnection', socket => received.push(socket))
const fakeSocket = {}
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
getMock().emit('secureConnection', fakeSocket)
assert.deepEqual(received, [fakeSocket])
return server.close()
})
})
it('forwards secureConnection to listeners registered after listen()', function () {
const { server, getMock } = createServerWithMock()
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
const received = []
server.on('secureConnection', socket => received.push(socket))
const fakeSocket = {}
getMock().emit('secureConnection', fakeSocket)
assert.deepEqual(received, [fakeSocket])
return server.close()
})
})
it('stops forwarding secureConnection after listener is removed', function () {
const { server, getMock } = createServerWithMock()
const received = []
const listener = socket => received.push(socket)
server.on('secureConnection', listener)
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
server.off('secureConnection', listener)
getMock().emit('secureConnection', {})
assert.deepEqual(received, [])
return server.close()
})
})
})
describe('.listen()', function () {
it('can use a host:port', function () {
let listenArgs
const mock = createMock({ address: '127.0.0.1', port: 8080 })
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function () {
assert.deepEqual(listenArgs, [8080, '127.0.0.1'])
return server.close()
})
})
it('can use a systemd socket', function () {
const SD_LISTEN_FDS_START = 3
const SOCKET_INDEX = 0
let listenArgs
const mock = createMock('http://<fd:3>')
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
const env = process.env
env.LISTEN_FDS = String(SOCKET_INDEX + 1)
env.LISTEN_PID = String(process.pid)
return server.listen({ systemdSocket: SOCKET_INDEX }).then(function () {
delete env.LISTEN_FDS
delete env.LISTEN_PID
assert.deepEqual(listenArgs, [{ fd: SOCKET_INDEX + SD_LISTEN_FDS_START }])
return server.close()
})
})
it('can use a socket', function () {
const socketPath = '/tmp/http-server-plus-test.sock'
let listenArgs
const mock = createMock(socketPath)
mock.listen = function (...args) {
listenArgs = args
setImmediate(() => this.emit('listening'))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ socket: socketPath }).then(function () {
assert.deepEqual(listenArgs, [resolvePath(socketPath)])
return server.close()
})
})
it('can use a HTTPS certificate', function () {
let capturedOpts
const mock = createMock({ address: '127.0.0.1', port: 8443 })
const server = HttpServerPlus.create({
createSecureServer: opts => {
capturedOpts = opts
return mock
},
})
return server.listen({ cert: 'my-cert', key: 'my-key', port: 8443 }).then(function (address) {
assert.equal(capturedOpts.cert, 'my-cert')
assert.equal(capturedOpts.key, 'my-key')
assert.ok(address.startsWith('https://'), `expected https URL, got: ${address}`)
return server.close()
})
})
it('returns a promise which will resolve once listening', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(function (address) {
assert.equal(typeof address, 'string')
assert.ok(address.startsWith('http://'), `expected http URL, got: ${address}`)
return server.close()
})
})
it('returns a promise which will reject if failure', function () {
const listenError = new Error('EADDRINUSE')
const mock = createMock(null)
mock.listen = function () {
setImmediate(() => this.emit('error', listenError))
}
const server = HttpServerPlus.create({ createServer: () => mock })
return server.listen({ port: 8080 }).then(
function () {
assert.fail('should have rejected')
},
function (err) {
assert.equal(err, listenError)
assert.equal(typeof err.niceAddress, 'string')
}
)
})
})
describe('.close()', function () {
it('closes all servers', function () {
const closed = []
let callCount = 0
const server = HttpServerPlus.create({
createServer: () => {
const id = callCount++
const mock = createMock({ address: '127.0.0.1', port: id })
mock.close = function () {
closed.push(id)
setImmediate(() => this.emit('close'))
}
return mock
},
})
return Promise.all([server.listen({ port: 0 }), server.listen({ port: 0 })])
.then(function () {
return server.close()
})
.then(function () {
assert.equal(closed.length, 2)
})
})
it('emit the `close` event when all servers are closed', function () {
const mock = createMock({ address: '127.0.0.1', port: 8080 })
const server = HttpServerPlus.create({ createServer: () => mock })
let closeEmitted = false
server.on('close', () => {
closeEmitted = true
})
return server
.listen({ port: 8080 })
.then(function () {
return server.close()
})
.then(function () {
assert.ok(closeEmitted)
})
})
it('emit the `close` event even if there are no servers', function () {
const server = HttpServerPlus.create()
// close() with no servers returns Promise.resolve() immediately (before emitting
// 'close' via setImmediate), so we must wait for the event itself, not the promise.
return new Promise(function (resolve) {
server.close(resolve)
})
})
})
})

View File

@@ -1,38 +0,0 @@
{
"name": "@vates/http-server-plus",
"version": "2.0.2",
"license": "ISC",
"description": "Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance",
"keywords": [
"http",
"https",
"server",
"ports"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/http-server-plus",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/http-server-plus",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20"
},
"dependencies": {
"promise-toolbox": "^0.21.0"
},
"devDependencies": {
"eslint-plugin-n": "^15.1.0"
},
"scripts": {
"pretest": "eslint .",
"test": "node --test index.spec.js",
"postversion": "npm publish --access public"
},
"private": false,
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
}
}

View File

@@ -19,12 +19,6 @@ export class NbdDisk extends RandomAccessDisk {
/** @type {number} */
#blockSize
/** @type {number | undefined} */
#hasBlockCursor
/** @type {number | undefined} */
#hasBlockPreviousIndex
constructor(nbdInfos, blockSize, { dataMap } = {}) {
super()
this.#blockSize = blockSize
@@ -33,38 +27,11 @@ export class NbdDisk extends RandomAccessDisk {
}
#processDatamap(rawDataMap) {
const ranges = rawDataMap
.filter(({ type, length }) => type === 0 && length > 0)
return rawDataMap
.filter(({ type }) => type === 0)
.map(({ offset, length }) => ({ offset, length }))
.sort(({ offset: offset1 }, { offset: offset2 }) => offset1 - offset2)
// hasBlock()'s forward-only cursor is only correct if the extents are
// sorted AND disjoint: an earlier extent must never reach into a block
// after a later one, otherwise the cursor could skip it and wrongly report
// a block as empty, dropping data and corrupting the output.
// Touching extents (cur.offset === prevEnd) are merged
const merged = []
for (const range of ranges) {
const last = merged[merged.length - 1]
if (last !== undefined) {
const lastEnd = last.offset + last.length
if (range.offset < lastEnd) {
throw new Error(
`overlapping ranges in data map: [${last.offset}, ${lastEnd}) and [${range.offset}, ${range.offset + range.length})`
)
}
if (range.offset === lastEnd) {
// touching: extend the previous extent
last.length += range.length
continue
}
}
// gap (or first range): new extent
merged.push(range)
}
return merged
}
/**
* @param {number} index
* @returns {Promise<DiskBlock>}
@@ -150,34 +117,6 @@ export class NbdDisk extends RandomAccessDisk {
return [...indexes].sort((a, b) => a - b)
}
/**
* Counts the allocated blocks without materializing the index list.
*
* @returns {number}
*/
getBlockIndexesCount() {
if (!this.#dataMap) {
throw new Error("can't getBlockIndexesCount before init")
}
const blockSize = this.getBlockSize()
let count = 0
let lastCountedBlock = -1
for (const { offset, length } of this.#dataMap) {
const firstBlockIndex = Math.floor(offset / blockSize)
const lastBlockIndex = Math.floor((offset + length - 1) / blockSize)
const from = Math.max(firstBlockIndex, lastCountedBlock + 1)
if (lastBlockIndex >= from) {
count += lastBlockIndex - from + 1
lastCountedBlock = lastBlockIndex
}
}
return count
}
/**
* @param {number} index
* @returns {boolean}
@@ -186,32 +125,8 @@ export class NbdDisk extends RandomAccessDisk {
if (!this.#dataMap) {
throw new Error("can't hasBlock before init")
}
const blockSize = this.getBlockSize()
const blockStart = index * blockSize
const blockEnd = blockStart + blockSize
let startExtentIndex = 0
if (this.#hasBlockCursor !== undefined && index >= this.#hasBlockPreviousIndex) {
startExtentIndex = this.#hasBlockCursor
} else {
this.#hasBlockCursor = undefined
}
this.#hasBlockPreviousIndex = index
const dataMap = this.#dataMap
const l = dataMap.length
for (let i = startExtentIndex; i < l; i++) {
const { offset, length } = dataMap[i]
if (offset >= blockEnd) {
// extents are sorted: nothing from here on can overlap this block
break
}
if (offset + length > blockStart) {
this.#hasBlockCursor = i
return true
}
}
return false
const blockStart = index * this.getBlockSize()
const blockEnd = (index + 1) * this.getBlockSize()
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && blockEnd >= offset)
}
}

View File

@@ -1,127 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert'
import { NbdDisk } from './NbdDisk.mjs'
const KiB = 1024
const MiB = 1024 * KiB
const TiB = 1024 * 1024 * MiB
const SOURCE_BLOCK_SIZE = 2 * MiB // VHD_BLOCK_SIZE, the block size used when importing from VMware
const CLUSTER_SIZE = 64 * KiB // qcow2 cluster size
// hasBlock() only needs the dataMap (passed through the constructor) and the
// block size, so we can exercise it without a live NBD connection.
describe('NbdDisk.hasBlock', () => {
it('reports allocated and unallocated blocks correctly', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(1), false)
assert.strictEqual(disk.hasBlock(4), false)
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(6), false)
})
it('handles repeated and out-of-order queries with the resume cursor', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
// repeated indexes (the qcow2 path queries each 2MB block ~32 times as it
// walks 64KB clusters) must return a stable answer
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(5), true)
assert.strictEqual(disk.hasBlock(6), false)
assert.strictEqual(disk.hasBlock(6), false)
// querying backwards must still be correct (falls back to a full scan)
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(5), true)
})
it('invalidates the cursor on a backward miss (no stale skip)', () => {
const dataMap = [
{ type: 0, offset: 3 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 3
{ type: 0, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 10
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(10), true) // forward hit: cursor parks on the high extent
assert.strictEqual(disk.hasBlock(1), false) // backward miss: must invalidate the cursor
assert.strictEqual(disk.hasBlock(3), true) // lower extent must not be skipped
})
it('throws on overlapping ranges (would break the resume cursor)', () => {
const dataMap = [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE },
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE }, // overlaps the previous range
]
assert.throws(() => new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap }), /overlapping ranges/)
})
it('merges touching ranges and drops zero-length / non type-0 entries', () => {
const dataMap = [
{ type: 0, offset: 2 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [2, 3)
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5)
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [1, 2)
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 }, // [.5, 1), touches [0, .5)
{ type: 1, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // wrong type: ignored
{ type: 0, offset: 7 * SOURCE_BLOCK_SIZE, length: 0 }, // zero length: ignored
]
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.hasBlock(0), true)
assert.strictEqual(disk.hasBlock(1), true)
assert.strictEqual(disk.hasBlock(2), true)
assert.strictEqual(disk.hasBlock(3), false)
assert.strictEqual(disk.hasBlock(7), false) // zero-length range dropped
assert.strictEqual(disk.hasBlock(10), false) // non type-0 range dropped
})
})
describe('NbdDisk.getBlockIndexesCount', () => {
const cases = {
'aligned, disjoint blocks': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
],
'a range spanning several blocks': [
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: 3 * SOURCE_BLOCK_SIZE }, // blocks 0,1,2,3
],
'two disjoint extents sharing block 0 (needs dedup)': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5) => block 0
{ type: 0, offset: 3 * (SOURCE_BLOCK_SIZE / 4), length: SOURCE_BLOCK_SIZE }, // [.75, 1.75) => blocks 0,1
],
'touching sub-block fragments': [
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 },
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 },
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE },
],
empty: [],
}
for (const [name, dataMap] of Object.entries(cases)) {
it(`matches getBlockIndexes().length: ${name}`, () => {
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
})
}
it('matches getBlockIndexes().length on the large sparse map', () => {
const virtualSize = Math.floor(3.5 * TiB)
const NB_RANGES = 113528
const dataMap = []
for (let i = 0; i < NB_RANGES; i++) {
dataMap.push({ type: 0, offset: Math.floor((i / NB_RANGES) * virtualSize), length: CLUSTER_SIZE })
}
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
})
})

View File

@@ -353,7 +353,6 @@ export default class NbdClient {
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
return pRetry(() => this.#readBlock(index, size), {
tries: this.#readBlockRetries,
when: error => error.code !== 'ERR_ABORTED',
onRetry: async err => {
warn('will retry reading block ', index, err)
await this.reconnect()
@@ -371,7 +370,7 @@ export default class NbdClient {
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type ( 0 means data).
*/
/* async */ getMap(signal) {
/* async */ getMap() {
return new Promise((resolve, reject) => {
const process = spawn('nbdinfo', [
'--json',
@@ -382,17 +381,8 @@ export default class NbdClient {
let errText = ''
process.stdout.on('data', data => (text += data))
process.stderr.on('data', data => (errText += data))
const onAbort = () => {
process.kill()
reject(signal.reason)
}
signal?.addEventListener('abort', onAbort, { once: true })
process.on('error', err => {
signal?.removeEventListener('abort', onAbort)
reject(err)
})
process.on('error', reject)
process.on('close', code => {
signal?.removeEventListener('abort', onAbort)
if (code !== 0) {
return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
}

View File

@@ -7,7 +7,6 @@ const { warn } = createLogger('vates:nbd-client:multi')
export default class MultiNbdClient {
#clients = []
#nbdConcurrency
#nextClient = 0
#options
#readAhead
#settings
@@ -98,31 +97,8 @@ export default class MultiNbdClient {
* @returns {Promise<Buffer>}
*/
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
const clientId = this.#nextClient++ % this.#clients.length
const client = this.#clients[clientId]
try {
return await client.readBlock(index, size)
} catch (err) {
// client.readBlock() already exhausted its own retries/reconnects: this connection is dead.
// Evict it so future reads stop being routed to it, and retry this read on a surviving
// client, since the data is usually still reachable through the others.
this.#evict(client)
if (this.#clients.length === 0) {
throw err
}
warn(`evicted a dead nbd client, retrying block ${index} on a remaining client`, { err })
return this.readBlock(index, size)
}
}
// no-op if `client` was already evicted by a concurrent failed read on the same client
#evict(client) {
const i = this.#clients.indexOf(client)
if (i === -1) {
return
}
this.#clients.splice(i, 1)
client.disconnect().catch(() => {})
const clientId = index % this.#clients.length
return this.#clients[clientId].readBlock(index, size)
}
/**
@@ -162,8 +138,8 @@ export default class MultiNbdClient {
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type (0 means no data).
*/
async getMap(signal) {
async getMap() {
// ask the map from one of the connected client
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap(signal)
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap()
}
}

View File

@@ -1,130 +0,0 @@
import { describe, it, mock } from 'node:test'
import assert from 'node:assert'
// MultiNbdClient instantiates NbdClient itself (`import NbdClient from './index.mjs'`), so the
// only way to control individual connections from a unit test is to replace that module with a
// fake before importing multi.mjs.
const created = []
class FakeNbdClient {
constructor(nbdInfo) {
this.nbdInfo = nbdInfo
this.disconnected = false
// overridden per-test after connect() to script this instance's behavior
this.readBlock = async () => {
throw new Error('readBlock not configured for this fake client')
}
created.push(this)
}
async connect() {}
async disconnect() {
this.disconnected = true
}
async getMap() {
return []
}
}
mock.module('./index.mjs', {
defaultExport: FakeNbdClient,
})
const { default: MultiNbdClient } = await import('./multi.mjs')
// a single settings entry is enough: connect() reuses candidates across concurrent slots, so
// nbdConcurrency still yields that many distinct FakeNbdClient instances
async function connectWithFakes(nbdConcurrency) {
created.length = 0
const client = new MultiNbdClient({ address: 'fake-host' }, { nbdConcurrency })
await client.connect()
return { client, fakes: created.slice() }
}
describe('MultiNbdClient.readBlock eviction', () => {
it('evicts a client whose readBlock rejects and retries on a surviving client', async () => {
const { client, fakes } = await connectWithFakes(2)
const [dead, alive] = fakes
dead.readBlock = async () => {
throw new Error('connection is dead')
}
alive.readBlock = async index => Buffer.from([index])
// index 0 initially routes to `dead` (0 % 2 === 0)
const data = await client.readBlock(0)
assert.deepStrictEqual(data, Buffer.from([0]))
assert.strictEqual(dead.disconnected, true, 'the dead client should have been disconnected')
// future reads must no longer be routed to the evicted client: with only `alive` left,
// every index should now succeed through it
alive.readBlock = async index => Buffer.from([index])
const data2 = await client.readBlock(1)
assert.deepStrictEqual(data2, Buffer.from([1]))
})
it('rejects once every client has been evicted, instead of retrying forever', async () => {
const { client, fakes } = await connectWithFakes(2)
const boom = new Error('connection is dead')
for (const fake of fakes) {
fake.readBlock = async () => {
throw boom
}
}
await assert.rejects(() => client.readBlock(0), boom)
assert.ok(
fakes.every(fake => fake.disconnected),
'every client should have been evicted (and disconnected) before giving up'
)
})
it('evicting the same dead client from concurrent failed reads is idempotent', async () => {
const { client, fakes } = await connectWithFakes(2)
const [dead, alive] = fakes
dead.readBlock = async () => {
throw new Error('connection is dead')
}
alive.readBlock = async index => Buffer.from([index])
let disconnectCalls = 0
dead.disconnect = async () => {
disconnectCalls++
dead.disconnected = true
}
// routing is round-robin by call order (0, 1, 0) over a 2-client pool: the 1st and 3rd
// calls both land on `dead` before either failure has been handled, so they race to evict
// the same client. The exact interleaving isn't guaranteed, only that eviction is safe
// either way and every read still completes.
const [data0, data1, data2] = await Promise.all([client.readBlock(10), client.readBlock(11), client.readBlock(12)])
assert.deepStrictEqual(data0, Buffer.from([10]))
assert.deepStrictEqual(data1, Buffer.from([11]))
assert.deepStrictEqual(data2, Buffer.from([12]))
assert.strictEqual(disconnectCalls, 1, 'the dead client must only be evicted/disconnected once')
})
})
describe('MultiNbdClient.readBlock routing', () => {
it('distributes reads round-robin by call order, not by index residue', async () => {
const { client, fakes } = await connectWithFakes(4)
const readsPerClient = fakes.map(() => 0)
fakes.forEach((fake, i) => {
fake.readBlock = async index => {
readsPerClient[i]++
return Buffer.from([index])
}
})
// a CBT-style changed-block set striding by exactly `nbdConcurrency`: under the old
// `index % clients.length` routing this would collapse entirely onto a single client
const stridedIndexes = Array.from({ length: 20 }, (_, i) => i * 4)
for (const index of stridedIndexes) {
await client.readBlock(index)
}
assert.deepStrictEqual(readsPerClient, [5, 5, 5, 5])
})
})

View File

@@ -13,19 +13,19 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.5.1",
"version": "3.2.2",
"engines": {
"node": ">=14.0"
},
"main": "./index.mjs",
"dependencies": {
"@vates/async-each": "^1.0.3",
"@vates/async-each": "^1.0.1",
"@vates/read-chunk": "^1.2.1",
"@xen-orchestra/async-map": "^0.1.4",
"@xen-orchestra/disk-transform": "^1.3.3",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/disk-transform": "^1.2.1",
"@xen-orchestra/log": "^0.7.1",
"promise-toolbox": "^0.21.0",
"xen-api": "^4.9.0"
"xen-api": "^4.7.6"
},
"devDependencies": {
"tap": "^18.7.0",
@@ -33,7 +33,6 @@
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node --experimental-test-module-mocks --test",
"test-integration": "tap --allow-incomplete-coverage tests/*.integ.mjs"
"test-integration": "tap --allow-incomplete-coverage"
}
}

View File

@@ -153,13 +153,7 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
nbdServer = await spawnNbdKit(path)
}
}
// reading past the end of the export must reject: a single client so the failure isn't
// multiplied by nbdConcurrency evicting and retrying on every other (equally healthy, equally
// unable to satisfy this out-of-range request) client
const singleClient = new MultiNbdClient(connectionSettings, { nbdConcurrency: 1, readAhead: 2 })
await singleClient.connect()
await assert.rejects(() => singleClient.readBlock(100, CHUNK_SIZE))
await singleClient.disconnect()
assert.rejects(() => client.readBlock(100, CHUNK_SIZE))
await client.disconnect()
// double disconnection shouldn't pose any problem

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/node-vsphere-soap",
"version": "2.1.3",
"version": "2.1.2",
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
"main": "lib/client.mjs",
"author": "reedog117",
@@ -11,8 +11,8 @@
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/log": "^0.7.2",
"axios": "^1.15.0",
"@xen-orchestra/log": "^0.7.1",
"axios": "^1.8.0",
"soap": "^1.0.0",
"soap-cookie": "^0.10.1"
},

View File

@@ -23,7 +23,7 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
Object.defineProperty(taskLog, '$root', { value: taskLog })
// start of a root task
onRootTaskStart(taskLog, event)
onRootTaskStart(taskLog)
} else {
// start of a subtask
const parent = taskLogs.get(parentId)
@@ -54,10 +54,10 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
}
if (type === 'end' && taskLog.$root === taskLog) {
onRootTaskEnd(taskLog, event)
onRootTaskEnd(taskLog)
}
}
onTaskUpdate(taskLog, event)
onTaskUpdate(taskLog)
}
}

View File

@@ -7,27 +7,6 @@ function alreadyEnded() {
throw new Error('task has already ended')
}
const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
// Create a serializable object from an error.
//
// Otherwise some fields might be non-enumerable and missing from logs.
const serializeError = error => {
return error instanceof Error
? {
...error, // Copy enumerable properties.
cause: error.cause,
code: error.code,
errors: serializeErrors(error.errors), // supports AggregateError
message: error.message,
name: error.name,
stack: error.stack,
}
: error
}
exports.serializeError = serializeError
// define a read-only, non-enumerable, non-configurable property
function define(object, property, value) {
Object.defineProperty(object, property, { value })
@@ -93,11 +72,7 @@ class Task {
_onProgress
get id() {
// Use a compact, sortable, string representation of the creation date,
// and add a random part to differentiate tasks created at the same time
//
// Due to the padding, dates are sortable up to 5188-04-22T11:04:28.415Z
return (this.id = Date.now().toString(36).padStart(9, '0') + '-' + Math.random().toString(36).slice(2))
return (this.id = Math.random().toString(36).slice(2))
}
set id(value) {
define(this, 'id', value)
@@ -161,10 +136,6 @@ class Task {
assert.equal(this.#status, PENDING)
assert.equal(this.#running, false)
if (result instanceof Error && result.toJSON === undefined) {
result = serializeError(result)
}
this.#emit('end', { status, result })
this._onProgress = alreadyEnded
this.#status = status

View File

@@ -4,7 +4,7 @@ const assert = require('node:assert').strict
const { describe, it } = require('node:test')
const { makeOnProgress } = require('./combineEvents.js')
const { Task, serializeError } = require('./index.js')
const { Task } = require('./index.js')
const noop = Function.prototype
@@ -28,36 +28,6 @@ function createTask(opts) {
return task
}
describe('serializeError', function () {
it('serializes simple error', function () {
const err = new Error('test Error')
err.code = 404
const serialized = serializeError(err)
assert.equal(serialized.message, 'test Error')
assert.equal(serialized.name, 'Error')
assert.equal(serialized.code, 404)
assert.notEqual(serialized.stack, undefined)
})
it('serializes aggregate error', function () {
const err = new AggregateError([new Error('test error 1'), new Error('test error 2')], 'Error message', {
cause: 'broken',
})
const serialized = serializeError(err)
assert.equal(serialized.message, 'Error message')
assert.equal(serialized.name, 'AggregateError')
assert.equal(serialized.cause, 'broken')
assert.notEqual(serialized.stack, undefined)
assert.equal(serialized.errors.length, 2)
assert.equal(serialized.errors[0].name, 'Error')
assert.equal(serialized.errors[1].name, 'Error')
assert.equal(serialized.errors[0].message, 'test error 1')
assert.equal(serialized.errors[1].message, 'test error 2')
assert.notEqual(serialized.errors[0].stack, undefined)
assert.notEqual(serialized.errors[1].stack, undefined)
})
})
describe('Task', function () {
describe('constructor', function () {
it('data properties are passed to the start event', async function () {
@@ -136,7 +106,7 @@ describe('Task', function () {
assert.equal(task.$events.length, 3)
assertEvent(task, { type: 'start' }, 0)
assertEvent(task, { type: 'abortionRequested', reason }, 1)
assertEvent(task, { type: 'end', status: 'failure', result: serializeError(result) }, 2)
assertEvent(task, { type: 'end', status: 'failure', result }, 2)
})
it('does not abort if the task succeed', async function () {
@@ -206,7 +176,7 @@ describe('Task', function () {
assertEvent(task, {
status: 'failure',
result: serializeError(error),
result: error,
type: 'end',
})
})
@@ -505,7 +475,7 @@ describe('Task', function () {
assert.equal(task.status, 'failure')
assertEvent(task, {
status: 'failure',
result: serializeError(e),
result: e,
type: 'end',
})
})
@@ -541,7 +511,7 @@ describe('Task', function () {
assert.equal(task.status, 'failure')
assertEvent(task, {
status: 'failure',
result: serializeError(e),
result: e,
type: 'end',
})
})

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.7.0",
"version": "0.6.2",
"engines": {
"node": ">=14"
},

View File

@@ -1,7 +1,7 @@
{
"name": "@vates/types",
"private": false,
"version": "2.0.1",
"version": "1.16.0",
"main": "./dist/index.mjs",
"exports": {
".": {
@@ -9,7 +9,6 @@
"types": "./dist/index.d.mts"
},
"./xo": "./dist/xo.d.mts",
"./xo-app": "./dist/xo-app.d.mts",
"./xen-api": "./dist/xen-api.d.mts",
"./common": {
"default": "./dist/common.mjs",
@@ -32,8 +31,7 @@
"devDependencies": {
"@eslint/js": "^9.19.0",
"typescript": "~5.6.3",
"typescript-eslint": "^8.61.0",
"@types/express": "^5.0.0"
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",

View File

@@ -1,4 +1,4 @@
import type { XoHost, XoNetwork, XoVif } from './xo.mjs'
import type { XoHost } from './xo.mjs'
declare const __brand: unique symbol
@@ -740,32 +740,3 @@ export type XapiVmStatsRaw = {
export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>
export type TrafficRuleProtocol = (typeof TRAFFIC_RULE_PROTOCOLS)[number]
export const SDN_CONTROLLER_OF_RULES_KEY = 'xo:sdn-controller:of-rules'
export const SDN_CONTROLLER_OF_METHOD_KEY = 'xo:sdn-controller:of-method'
export const SDN_CONTROLLER_OF_FORMAT_KEY = 'xo:sdn-controller:of-format'
export const TRAFFIC_RULE_PROTOCOLS = ['ARP', 'ICMP', 'IP', 'TCP', 'UDP'] as const
export const TRAFFIC_RULE_PROTOCOLS_WITH_PORT: readonly TrafficRuleProtocol[] = ['TCP', 'UDP']
export type TrafficRuleTargetType = XoNetwork['type'] | XoVif['type']
export type TrafficRuleDirection = 'from' | 'to' | 'from/to'
export type RawTrafficRule = {
allow: boolean
protocol: TrafficRuleProtocol
ipRange: string
direction: TrafficRuleDirection
port?: string
}
type BaseTrafficRule = RawTrafficRule & {
id: string
networkId: XoNetwork['id']
}
export type TrafficRule =
| (BaseTrafficRule & { type: XoVif['type']; sourceId: XoVif['id'] })
| (BaseTrafficRule & { type: XoNetwork['type']; sourceId: XoNetwork['id'] })

View File

@@ -1,8 +1,5 @@
export * from './common.mjs'
export * from './xen-api.mjs'
export * from './xo.mjs'
export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/xen-orchestra-acl.mjs'
export * from './lib/complex-matcher.mjs'
export * from './lib/rest-api.mjs'

View File

@@ -1,96 +0,0 @@
import type { NextFunction, Request, Response } from 'express'
import type { VatesTask } from './vates-task.mjs'
import type { XoApp } from '../xo-app.mjs'
export type SecurityName = '*' | 'token' | 'basic' | 'none'
export type FieldDefinition = { optional?: boolean; nullable?: boolean } & (
| {
type: 'string'
example?: string
}
| {
type: 'boolean'
example?: boolean
}
| {
type: 'number'
example?: number
}
| {
type: 'enum'
enum: string[]
example?: string
}
| {
type: 'object'
fields: Record<string, FieldDefinition>
}
| {
type: 'array'
items: FieldDefinition
example?: unknown[]
}
)
export type ParamFieldDefinition = Exclude<
FieldDefinition,
{ type: 'boolean' } | { type: 'object' } | { type: 'array' }
>
export type QueryFieldDefinition = Exclude<FieldDefinition, { type: 'object' } | { type: 'array' }>
type RestApi = object & { xoApp: XoApp }
export type CreateAction = <Result>(
callback: (task: VatesTask) => MaybePromise<Result>,
opts: {
sync?: boolean
statusCode?: number
taskProperties: { name: string; [key: string]: unknown }
}
) => Promise<Result | undefined>
export interface BaseRouteDefinition<Middleware> {
method: 'get' | 'post' | 'put' | 'delete' | 'patch'
endpoint: string
description?: string
tags?: string[]
params?: Record<string, ParamFieldDefinition>
query?: Record<string, QueryFieldDefinition>
body?: Record<string, FieldDefinition>
responses?: Array<{
status: number
description: string
schema?: Record<string, FieldDefinition>
}>
middlewares?: Middleware[]
scope?: 'acl'
callback: (params: {
req: Request
res: Response
next: NextFunction
restApi: RestApi
createAction: CreateAction
}) => MaybePromise<unknown>
security?: SecurityName
}
export type MiddlewareDescriptor =
| { name: 'json' | 'urlencoded' | 'text' | 'raw'; options?: Record<string, unknown> }
| { name: 'acl'; acls: LooseAclEntry | LooseAclEntry[] }
type MaybePromise<T> = T | Promise<T>
/**
* Loosely-typed mirror of {@link https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts | AclEntry}
* from `@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts`.
*/
export type LooseAclEntry = {
resource: string
action?: string | ((opts: { req: object; restApi: RestApi }) => string | undefined)
actions?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
objectId?: string | ((opts: { req: object; restApi: RestApi }) => string)
objectIds?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
object?: object | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object> | undefined)
objects?: object[] | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object[]> | undefined)
getObject?: (opts: { restApi: RestApi }) => (id: string) => MaybePromise<object>
}
export type PluginRestRouteDefinition = BaseRouteDefinition<MiddlewareDescriptor>

View File

@@ -1,83 +0,0 @@
import { Branded } from '../common.mjs'
import type { XoGroup, XoUser } from '../xo.mjs'
type XoAclBaseRole = {
id: Branded<'acl-v2-role'>
name: string
description?: string
privilegeIds: XoAclBasePrivilege['id'][]
}
type XoAclRegularRole = XoAclBaseRole & {
groupIds: XoGroup['id'][]
userIds: XoUser['id'][]
}
type XoAclTemplateRole = XoAclBaseRole & {
isTemplate: true
roleTemplateId: number
}
export type XoAclRole = XoAclRegularRole | XoAclTemplateRole
export type XoAclSupportedActionsByResource = {
[resource: string]: Record<string, unknown>
}
/**
* E.g
* vms {
* shutdown: {
* clean: true,
* hard: true,
* }
* }
*
* `shutdown | shutdown:clean | shutdown:hard`
*/
export type GetKeysRecursively<T, Prefix extends string = ''> = {
[K in keyof T]: T[K] extends object
? K extends string
? `${Prefix}${K}` | GetKeysRecursively<T[K], `${Prefix}${K}:`>
: never
: K extends string
? `${Prefix}${K}`
: never
}[keyof T]
export type XoAclSupportedResource<TActionsByResource extends XoAclSupportedActionsByResource> =
keyof TActionsByResource
export type XoAclSupportedActions<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = (GetKeysRecursively<TActionsByResource[TResource]> & string) | '*'
export type XoAclBasePrivilege = {
id: Branded<'acl-v2-privilege'>
resource: string
action: string
selector?: string
effect: 'allow' | 'deny'
roleId: XoAclRole['id']
}
export type XoAclPrivilege<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = XoAclBasePrivilege & {
resource: TResource
action: XoAclSupportedActions<TActionsByResource, TResource>
}
export type XoUserRole = {
id: string
roleId: XoAclRole['id']
userId: XoUser['id']
}
export type XoGroupRole = {
id: string
roleId: XoAclRole['id']
groupId: XoGroup['id']
}

View File

@@ -1,103 +1,28 @@
import {
WrappedXenApiRecord,
XenApiHost,
XenApiHostWrapped,
XenApiNetwork,
XenApiNetworkWrapped,
XenApiRecord,
XenApiSr,
XenApiTask,
XenApiVbd,
XenApiVdi,
XenApiVif,
XenApiVm,
XenApiVmWrapped,
XenApiVtpm,
} from '../xen-api.mjs'
import type {
BOND_MODE,
OPAQUE_REF_NULL,
SUPPORTED_VDI_FORMAT,
VBD_MODE,
VBD_TYPE,
VDI_TYPE,
VIF_LOCKING_MODE,
VM_OPERATIONS,
} from '../common.mjs'
import type { PassThrough, Readable } from 'node:stream'
import type {
XapiXoRecord,
XoGpuGroup,
XoVgpuType,
XoHost,
XoNetwork,
XoPif,
XoSr,
XoUser,
XoVdi,
XoVgpuType,
XoVm,
XoVmTemplate,
XoVif,
XoVmSnapshot,
} from '../xo.mjs'
/**
* Properties accepted by {@link Xapi.editVm}.
*
* Field names match the canonical (camelCase) properties defined by the
* underlying `_editVm` (via `makeEditObject`). `_editVm` still resolves the
* snake_case and legacy aliases, but the canonical names are used here.
*/
export interface EditVmProps {
affinityHost?: string | null
autoPoweron?: boolean
blockedOperations?: Partial<Record<VM_OPERATIONS, boolean | string | null>>
coresPerSocket?: number | string | null
cpuCap?: number | null
cpuMask?: number[]
cpuWeight?: number | null
cpus?: number
cpusStaticMax?: number | string
/**
* Update VM creation metadata stored under `other_config.xo:*`. The object is
* merged with the existing data.
*/
creation?: { user?: string }
expNestedHvm?: boolean
hasVendorDevice?: boolean
highAvailability?: 'best-effort' | 'restart' | ''
hvmBootFirmware?: string | null
memory?: number | string
memoryMax?: number | string
memoryMin?: number | string
memoryStaticMax?: number | string
nameDescription?: string
nameLabel?: string
nestedVirt?: boolean
nicType?: string | null
notes?: string | null
PV_args?: string
secureBoot?: boolean
startDelay?: number
suspendSr?: string | null
tags?: string[]
uefiMode?: 'setup' | 'user'
vga?: 'std' | 'cirrus'
videoram?: 1 | 2 | 4 | 8 | 16
viridian?: boolean
virtualizationMode?: 'pv' | 'hvm'
xenStoreData?: Record<string, string | null>
}
/** Properties accepted by {@link Xapi.editVif}. */
export interface EditVifProps {
ipv4Allowed?: string[]
ipv6Allowed?: string[]
lockingMode?: VIF_LOCKING_MODE
// in kB/s, `null` removes the limit
rateLimit?: number | null
txChecksumming?: boolean
}
import type { SUPPORTED_VDI_FORMAT } from '../common.mjs'
export type XcpPatches = {
changelog?: {
@@ -129,29 +54,14 @@ export type XsPatches = {
}
export interface Xapi {
status: string
pool?: { uuid: string }
sessionId: string
_auth: { user: string; password: string }
_url: { protocol: string; hostname: string; hostnameRaw: string; port?: string }
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
connectVif(vifId: XoVif['id']): Promise<void>
getField<T extends XenApiRecord, K extends keyof T>(
type: Extract<WrappedXenApiRecord, T>['$type'],
ref: T['$ref'],
field: K
): Promise<T[K]>
getObject: <
XoRecord extends XapiXoRecord,
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
>(
idOrUuidOrRef: XoRecord['id'] | WrappedRecord['$ref'] | WrappedRecord['uuid'],
defaultValue?: WrappedRecord
) => WrappedRecord
createNetwork(
params:
| {
@@ -168,101 +78,12 @@ export interface Xapi {
vlan: number
}
): Promise<XenApiNetworkWrapped>
createBondedNetwork(params: {
bondMode: BOND_MODE
pifIds: XoPif['id'][]
name: string
description?: string
mtu?: number
}): Promise<XenApiNetworkWrapped>
deleteNetwork(id: XoNetwork['id']): Promise<void>
deleteVif(vifId: XoVif['id']): Promise<void>
disconnectVif(vifId: XoVif['id']): Promise<void>
exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
emergencyShutdownHost(hostId: XoHost['id']): Promise<void>
ejectHostFromPool(hostId: XoHost['id']): Promise<void>
host_smartReboot(
hostRef: XenApiHost['$ref'],
bypassBlockedSuspend?: boolean,
bypassCurrentVmCheck?: boolean
): Promise<void>
migrateVm(
vmId: XoVm['id'],
hostXapi: Xapi,
hostId: XoHost['id'],
opts?: {
force?: boolean
mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
migrationNetworkId?: XoNetwork['id']
sr?: XoSr['id']
bypassAssert?: boolean
}
): Promise<void>
joinPool(masterAddress: string, masterUsername: string, masterPassword: string, force?: boolean): Promise<void>
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
pool_emergencyShutdown(): Promise<void>
resumeVm(id: XoVm['id']): Promise<void>
revertVm(snapshotId: XoVmSnapshot['id']): Promise<void>
unpauseVm(id: XoVm['id']): Promise<void>
cloneVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
fast?: boolean
}
): Promise<XenApiVmWrapped>
copyVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
srOrSrId?: XoSr['id'] | XoSr
}
): Promise<XenApiVmWrapped>
remoteCopyVm(
vmId: XoVm['id'],
targetXapi: Xapi,
targetSrId: XoSr['id'],
opts?: {
compress?: 'zstd' | 'gzip' | boolean
nameLabel?: string
}
): Promise<{ vm: XenApiVmWrapped }>
forgetSr(id: XoSr['id']): Promise<void>
forgetHost(hostId: XoHost['id']): Promise<void>
powerOnHost(hostId: XoHost['id']): Promise<void>
rebootHost(hostId: XoHost['id'], force?: boolean): Promise<void>
shutdownHost(hostId: XoHost['id'], opts?: { force?: boolean; bypassEvacuate?: boolean }): Promise<void>
SR_create(params: {
content_type?: XoSr['content_type']
// index signature, not `Record<string, string>`: tsoa emits an empty schema for `Record`
// (https://github.com/lukeautry/tsoa/pull/1858). device_config keys are driver-specific (SR_type).
device_config: {
chapuser?: string
chappassword?: string
device?: string
legacy_mode?: string
location?: string
nfsversion?: string
options?: string
password?: string
SCSIid?: string
server?: string
serverpath?: string
target?: string
targetIQN?: string
username?: string
} & {
[key: string]: string
}
host: XenApiHost['$ref']
name_description?: XoSr['name_description']
name_label: XoSr['name_label']
physical_size?: number
shared: boolean
sm_config?: { [key: string]: string }
type: XoSr['SR_type']
}): Promise<XenApiSr['$ref']>
SR_importVdi(
ref: XenApiSr['$ref'],
stream: Readable,
@@ -280,13 +101,6 @@ export interface Xapi {
xenstore_data?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
editVm(
id: XoVm['id'],
props: EditVmProps,
checkLimits?: (limits: Record<string, number>, vm: XenApiVmWrapped) => Promise<void>
): Promise<void>
editVif(id: XoVif['id'], props: EditVifProps): Promise<void>
startVm(
id: XoVm['id'],
opts?: {
@@ -304,7 +118,7 @@ export interface Xapi {
): Promise<void>
VM_export(
vmRef: XenApiVm['$ref'],
opts?: { cancelToken?: unknown; compress?: boolean | 'zstd' | 'gzip'; useSnapshot?: boolean }
opts?: { cancelToken?: unknown; compress?: boolean; useSnapshot?: boolean }
): ReturnType<Xapi['getResource']>
VM_import(
stream: Readable,
@@ -316,7 +130,6 @@ export interface Xapi {
metadataVm: {
affinityHost?: XoHost['id']
autoPoweron?: boolean
cpus?: number
memory?: number
name_description?: string
name_label: string
@@ -358,51 +171,12 @@ export interface Xapi {
vgpuType?: XoVgpuType['id']
gpuGroup?: XoGpuGroup['id']
copyHostBiosStrings?: boolean
highAvailability?: XoVm['high_availability']
hvmBootFirmware?: 'uefi' | 'bios'
secureBoot?: boolean
},
checkLimits?: boolean,
creatorId?: XoUser['id'],
opts?: { destroyAllVifs: boolean }
): Promise<XenApiVmWrapped>
moveVdi(
vdiId: XenApiVdi['$ref'] | XoVdi['id'],
srId: XenApiSr['$ref'] | XoSr['id'],
opts?: { _failOnCbtError?: boolean }
): Promise<XenApiVdi>
VBD_create(params: {
bootable?: boolean
empty?: boolean
mode?: VBD_MODE
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
type?: VBD_TYPE
unpluggable?: boolean
userdevice?: string
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
VM: XenApiVm['$ref']
}): Promise<XenApiVbd['$ref']>
VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
VDI_create(
options: {
name_description?: XoVdi['name_description']
name_label?: XoVdi['name_label']
other_config?: XoVdi['other_config']
read_only?: boolean
sharable?: boolean
SR?: XenApiSr['$ref']
tags?: XoVdi['tags']
type?: VDI_TYPE
virtual_size: XoVdi['size']
xenstore_data?: Record<string, string>
},
extraOptions?: {
sm_config?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
VDI_exportContent(
@@ -411,27 +185,9 @@ export interface Xapi {
): Promise<Readable & { length?: number }>
VDI_importContent(
vdiRef: XenApiVdi['$ref'],
stream: Readable | Buffer,
stream: Readable,
opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
): Promise<void>
VIF_create(
options: {
currently_attached?: boolean
device?: string
ipv4_allowed?: string[]
ipv6_allowed?: string[]
locking_mode?: VIF_LOCKING_MODE
MTU?: number
network: XenApiNetwork['$ref']
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
VM: XenApiVm['$ref']
},
extraOptions?: {
MAC?: string
}
): Promise<XenApiVif['$ref']>
VM_createCloudInitConfig(
vmRef: XenApiVm['$ref'],
cloudConfig: string,
@@ -450,29 +206,5 @@ export interface Xapi {
pathname: string,
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
): Promise<{ body: Readable }>
clearHost(
host: Pick<XenApiHostWrapped, '$ref' | '$pool'>,
force?: boolean,
opts?: { transient?: boolean }
): Promise<void>
disableHost(hostId: XoHost['id'], opts?: { transient?: boolean }): Promise<void>
enableHost(hostId: XoHost['id']): Promise<void>
getRecordByUuid<
Type extends WrappedXenApiRecord['$type'],
XenApiRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: Type }>,
>(
type: Type,
uuid: XenApiRecord['uuid']
): Promise<XenApiRecord>
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
destroySr(id: XoSr['id']): Promise<void>
xostor_delete(ref: XenApiSr['$ref']): Promise<void>
objects: {
indexes: {
type: {
[XenApiRecord in WrappedXenApiRecord as XenApiRecord['$type']]: Record<XenApiRecord['uuid'], XenApiRecord>
}
}
}
}

View File

@@ -82,7 +82,6 @@ type TagCallMethods = {
type WrapperXenApi<T, Type extends string, Fn = { (): void }> = T & {
$call: Fn
$callAsync: Fn
$pool: XenApiPool
$type: Type
$snapshot(params: {
cancelToken?: unknown
@@ -192,7 +191,6 @@ export interface XenApiPool {
ha_host_failures_to_tolerate: number
ha_overcommitted: boolean
ha_plan_exists_for: number
ha_reboot_vm_on_internal_shutdown?: boolean
ha_statefiles: string[]
health_check_config: Record<string, string>
igmp_snooping_enabled?: boolean
@@ -492,9 +490,7 @@ export interface XenApiDrTask {
uuid: string
}
type XenApiHostCallMethods = TagCallMethods & {
<T>(method: 'call_plugin', plugin: string, fn: string, args: Record<string, string>): Promise<T>
}
type XenApiHostCallMethods = TagCallMethods & {}
export interface XenApiHost {
$ref: Branded<'host'>
address: string
@@ -571,9 +567,7 @@ export interface XenApiHost {
uuid: string
virtual_hardware_platform_versions: number[]
}
export interface XenApiHostWrapped extends WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods> {
$restartAgent(): Promise<void>
}
export type XenApiHostWrapped = WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods>
export interface XenApiHostCrashdump {
$ref: Branded<'host_crashdump'>
@@ -628,9 +622,7 @@ export interface XenApiHostCpu {
vendor: string
}
type XenApiNetworkCallMethods = TagCallMethods & {
(method: 'add_purpose', purpose: NETWORK_PURPOSE): Promise<void>
}
type XenApiNetworkCallMethods = TagCallMethods & {}
export interface XenApiNetwork {
$ref: Branded<'network'>
@@ -1016,7 +1008,6 @@ export interface XenApiMessage {
timestamp: string
uuid: string
}
export type XenApiMessageWrapped = WrapperXenApi<XenApiMessage, 'message'>
export interface XenApiSecret {
$ref: Branded<'secret'>

View File

@@ -1,419 +0,0 @@
import { EventEmitter } from 'node:stream'
import type {
AnyXoJob,
AnyXoLog,
XapiXoRecord,
XoAuthenticationToken,
XoBackupRepository,
XoConfigBackupArchive,
XoGroup,
XoHost,
XoPool,
XoPoolBackupArchive,
XoProxy,
XoSchedule,
XoServer,
XoTask,
XoUser,
XoVif,
XoVm,
XoVmBackupArchive,
} from './xo.mjs'
import { VatesTask } from './lib/vates-task.mjs'
import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
import {
Xapi,
XapiHostStats,
XapiPoolStats,
XapiStatsGranularity,
XapiVmStats,
XenApiGpuGroupWrapped,
XenApiHostWrapped,
XenApiMessageWrapped,
XenApiNetworkWrapped,
XenApiPbdWrapped,
XenApiPciWrapped,
XenApiPgpuWrapped,
XenApiPifWrapped,
XenApiPoolWrapped,
XenApiSmWrapped,
XenApiSrWrapped,
XenApiVbdWrapped,
XenApiVdiWrapped,
XenApiVgpuTypeWrapped,
XenApiVgpuWrapped,
XenApiVifWrapped,
XenApiVmWrapped,
XenApiVtpmWrapped,
XoAclBasePrivilege,
XoAclRole,
XoGroupRole,
XoUserRole,
} from './index.mjs'
type FeatureCode =
| 'BACKUP.DELTA'
| 'BACKUP.DELTA_REPLICATION'
| 'BACKUP.FULL'
| 'BACKUP.HEALTHCHECK'
| 'BACKUP.METADATA'
| 'BACKUP.MIRROR'
| 'BACKUP.WITH_RAM'
| 'BACKUP.SMART_BACKUP'
| 'BACKUP.S3'
| 'DOCKER'
| 'EXPORT.XVA'
| 'LIST_MISSING_PATCHES'
| 'POOL_EMERGENCY_SHUTDOWN'
| 'RBAC'
| 'ROLLING_POOL_UPDATE'
| 'ROLLING_POOL_REBOOT'
| 'SMART_REBOOT'
| 'WARM_MIGRATION'
| 'PLUGIN.OPENMETRICS'
type XapiRecordByXapiXoRecord = {
gpuGroup: XenApiGpuGroupWrapped
host: XenApiHostWrapped
message: XenApiMessageWrapped
network: XenApiNetworkWrapped
PBD: XenApiPbdWrapped
PCI: XenApiPciWrapped
PGPU: XenApiPgpuWrapped
PIF: XenApiPifWrapped
pool: XenApiPoolWrapped
SR: XenApiSrWrapped
SM: XenApiSmWrapped
VBD: XenApiVbdWrapped
VDI: XenApiVdiWrapped
'VDI-snapshot': XenApiVdiWrapped
'VDI-unmanaged': XenApiVdiWrapped
vgpu: XenApiVgpuWrapped
vgpuType: XenApiVgpuTypeWrapped
VIF: XenApiVifWrapped
VM: XenApiVmWrapped
'VM-controller': XenApiVmWrapped
'VM-snapshot': XenApiVmWrapped
'VM-template': XenApiVmWrapped
VTPM: XenApiVtpmWrapped
}
type LicenseProductId =
| 'premium'
| 'xcpng-enterprise'
| 'xcpng-standard'
| 'xo-proxy'
| 'xosan.trial'
| 'xostor'
| 'xostor.trial'
type LicenseProductType = 'xo' | 'xoproxy' | 'xcpng' | 'xosan' | 'xostor'
type License = {
id: string
type: 'license'
created: number
licenseType: 'unit'
productId: LicenseProductId
tags: string[]
expires?: number
boundObjectId?: string
bindDate?: number
buyer?: { token: string; email: string }
history?: { date: number; boundObjectId: string }[]
rebound?: number
productTypes?: LicenseProductType[]
bundleInfo?: { name: string; id: string }
}
export type XoApp = {
hooks: EventEmitter
_redis: {
get(key: string): Promise<string | null>
mSet(args: string[]): Promise<unknown>
sMembers(key: string): Promise<string[]>
sIsMember(key: string, member: string): Promise<boolean>
}
config: {
get<T = string>(path: string): T
getOptional<T = unknown>(path: string): T | undefined
getOptionalDuration(path: string): number | undefined
getGuiRoutes(): Promise<{
default: {
url: string
path: string
}
v5?: {
url: string
path: string
}
v6?: {
url: string
path: string
}
[key: string]:
| undefined
| {
url: string
path: string
}
}>
}
objects: EventEmitter & {
allIndexes: {
type: {
getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
}
}
}
tasks: EventEmitter & {
abort(id: XoTask['id'], reason?: string): Promise<void>
clearLogs(): Promise<void>
create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
deleteLog(id: XoTask['id']): Promise<void>
get(id: XoTask['id']): Promise<XoTask>
list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
}
apiContext: {
user?: XoUser
permission?: XoUser['permission'] | 'none' | null
}
// methods ------------
allocIpAddresses(vifId: XoVif['id'], addAddresses?: string[], removeAddresses?: string[]): Promise<void>
addAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<XoGroupRole>
addAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<XoUserRole>
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
addApiMethod: <A extends unknown[], R>(
name: string,
method: (...args: A) => Promise<R>,
//we use any since it is a legacy call, might be better typed in another pr
info: {
resolve?: any
params?: any
}
) => () => void // eslint-disable-line @typescript-eslint/no-explicit-any
registerRestRoutes: (routes: PluginRestRouteDefinition[], base?: string) => () => void
authenticateUser: (
credentials: { token?: string; username?: string; password?: string },
userData?: { ip?: string },
opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
backupGuard(poolId: XoPool['id']): Promise<void>
/* Throw if no authorization */
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
/* validate, apply and persist the configuration of a plugin */
configurePlugin(id: string, configuration: unknown, mergeWithExisting?: boolean): Promise<void>
/* connect a server (XCP-ng/XenServer) */
connectXenServer(id: XoServer['id']): Promise<void>
// TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)
createAclV2Privilege(
privilege: Omit<XoAclBasePrivilege, 'id'>,
options?: {
force?: boolean
}
): Promise<XoAclBasePrivilege>
copyAclV2Role(
id: XoAclRole['id'],
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
): Promise<XoAclRole['id']>
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
createAuthenticationToken(opts: {
client?: {
id?: string
[key: string]: unknown
}
description?: string
expiresIn?: string | number
userId: XoUser['id']
}): Promise<XoAuthenticationToken>
createRemote(params: {
name: string
options?: string
proxy?: XoProxy['id']
url: string
}): Promise<XoBackupRepository>
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
deleteAclV2GroupRole(
groupId: XoGroup['id'],
roleId: XoAclRole['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<boolean>
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
detachHostFromPool(hostId: XoHost['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
/* disconnect a server (XCP-ng/XenServer) */
disconnectXenServer(id: XoServer['id']): Promise<void>
findEnabledScheduleSequenceFromSchedule(id: XoSchedule['id']): Promise<XoSchedule | undefined>
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
getAclV2RolePrivileges(roleId: XoAclRole['id']): Promise<XoAclBasePrivilege[]>
getAclV2Role(id: XoAclRole['id'], opts?: { bypassAuthorization?: boolean }): Promise<XoAclRole>
deleteAclV2UserRole(
userId: XoUser['id'],
roleId: XoAclRole['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<boolean>
getAclV2GroupRoles(
groupId: XoGroup['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<Exclude<XoAclRole, { isTemplate: true }>[]>
getAclV2Roles(): Promise<XoAclRole[]>
getAclV2UserPrivileges(userId: XoUser['id']): Promise<XoAclBasePrivilege[]>
getAclV2UserRoles(
userId: XoUser['id'],
opts?: { bypassAuthorization?: boolean; fromGroup?: boolean; fromUser?: boolean }
): Promise<XoAclRole[]>
getAllGroups(): Promise<XoGroup[]>
getAllProxies(): Promise<XoProxy[]>
getAllJobs<T extends AnyXoJob['type']>(type: T): Promise<Extract<AnyXoJob, { type: T }>[]>
getAllJobs(type?: string): Promise<AnyXoJob[]>
getProxy(id: XoProxy['id']): Promise<XoProxy>
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
getAllRemotes(): Promise<XoBackupRepository[]>
removeRemote(id: XoBackupRepository['id']): Promise<void>
getAllRemotesInfo(): Promise<
Record<
XoBackupRepository['id'],
{
size?: number
used?: number
available?: number
encryption?: {
algorithm: string
isLegacy: boolean
recommanded: string
}
}
>
>
getAllSchedules(): Promise<XoSchedule[]>
getAllUsers(): Promise<XoUser[]>
getAllXenServers(): Promise<XoServer[]>
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
getBackupNgLogsSorted(opts: {
after?: number
before?: number
filter: (log: AnyXoLog) => boolean
limit?: number
}): Promise<AnyXoLog[]>
getGroup(id: XoGroup['id']): Promise<XoGroup>
getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
getObjectsByType: <T extends XapiXoRecord>(
type: T['type'],
opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
) => Record<T['id'], T> | undefined
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
getUser: (id: XoUser['id']) => Promise<XoUser>
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
getXapiObject: <T extends XapiXoRecord>(
maybeId: T['id'] | T,
type: T['type'] | T['type'][]
) => XapiRecordByXapiXoRecord[T['type']]
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
getXenServer(id: XoServer['id']): Promise<XoServer>
hasFeatureAuthorization(featureCode: string): Promise<boolean>
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
}>
/** `null` when the listing of a backup repository failed */
listVmBackupsNg(
backupRepositoryIds: XoBackupRepository['id'][],
opts?: { _forceRefresh?: boolean; vmId?: XoVm['id'] }
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]> | null>>
pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
/** Allow to add a new server in the DB (XCP-ng/XenServer) */
registerXenServer(
body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
allowUnauthorized?: XoServer['allowUnauthorized']
password: string
readOnly?: XoServer['readOnly']
}
): Promise<XoServer>
rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask; shutdownPinnedVms?: boolean }): Promise<void>
rollingPoolUpdate(
pool: XoPool,
opts?: { rebootVm?: boolean; parentTask?: VatesTask; shutdownPinnedVms?: boolean }
): Promise<void>
setVmResourceSet(vmId: XoVm['id'], resourceSetId: string | null, force?: boolean): Promise<void>
shareVmResourceSet(vmId: XoVm['id']): Promise<void>
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
runJob(job: AnyXoJob, schedule: XoSchedule): void
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
testRemote(
id: XoBackupRepository['id']
): Promise<
| { success: true; readRate: number; writeRate: number }
| { success: false; step: string; file: string; error: unknown }
>
/** Remove a server from the DB (XCP-ng/XenServer) */
unregisterXenServer(id: XoServer['id']): Promise<void>
updateUser(
id: XoUser['id'],
updates: {
/**
* @deprecated
*/
email?: string
authProviders?: Record<string, string>
name?: string
password?: string
permission?: string
preferences?: Record<string, string>
}
): Promise<void>
updateAclV2Privilege(
privilegeId: XoAclBasePrivilege['id'],
privilege: XoAclBasePrivilege
): Promise<XoAclBasePrivilege>
updateAclV2Role(
roleId: XoAclRole['id'],
role: {
name?: XoAclRole['name']
description?: XoAclRole['description'] | null
},
options?: {
force?: boolean
}
): Promise<XoAclRole>
updateGroup(
id: XoGroup['id'],
updates: {
name?: string
}
): void
updateRemote(
id: XoBackupRepository['id'],
params: {
enabled?: boolean
name?: string
options?: string | null
proxy?: XoProxy['id'] | null
url?: string
}
): Promise<XoBackupRepository>
getAllXapis(): Record<string, Xapi>
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
getLicenses(params?: { productType?: LicenseProductType }): Promise<License[]>
bindLicense(params: { licenseId: string; boundObjectId: string }): Promise<License>
unbindLicense(params: {
productId: LicenseProductId
boundObjectId: string
licenseId: string
data?: Record<string, string>
}): Promise<License>
}

View File

@@ -19,7 +19,6 @@ import type {
VM_POWER_STATE,
} from './common.mjs'
import type * as CMType from './lib/complex-matcher.mjs'
import { XoAclBasePrivilege, XoAclRole } from './lib/xen-orchestra-acl.mjs'
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
type BaseXapiXo = {
@@ -76,7 +75,7 @@ type BaseXoVm = BaseXapiXo & {
expNestedHvm: boolean
isNestedVirtEnabled: boolean
hasVendorDevice: boolean
high_availability: 'best-effort' | 'restart' | ''
high_availability: string
installTime?: number | null
isFirmwareSupported: boolean
memory: {
@@ -94,7 +93,7 @@ type BaseXoVm = BaseXapiXo & {
notes?: string
os_version: Record<string, string> | null
other: Record<string, string>
parent?: XoVmSnapshot['id']
parent?: XoVm['id']
power_state: VM_POWER_STATE
pvDriversDetected?: boolean
pvDriversUpToDate?: boolean
@@ -107,10 +106,6 @@ type BaseXoVm = BaseXapiXo & {
suspendSr?: XoSr['id']
tags: string[]
vga?: string
/**
* Value in MiB.
* See: https://wiki.xenproject.org/wiki/XCP_PV_templates_start
*/
videoram?: number
viridian: boolean
virtualizationMode: DOMAIN_TYPE
@@ -222,7 +217,6 @@ export type XoGroup = {
provider?: string
providerGroupId?: string
users: XoUser['id'][]
aclRoleIds: XoAclRole['id'][]
}
export type XoHost = BaseXapiXo & {
@@ -336,7 +330,6 @@ export type XoNetwork = BaseXapiXo & {
defaultIsLocked: boolean
id: Branded<'network'>
insecureNbd?: boolean
isBonded: boolean
name_description: string
name_label: string
nbd?: boolean
@@ -434,7 +427,6 @@ export type XoPool = BaseXapiXo & {
current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
default_SR?: XoSr['id']
HA_enabled: boolean
haRebootVmOnInternalShutdown: boolean
haSrs: XoSr['id'][]
id: Branded<'pool'>
master: XoHost['id']
@@ -566,18 +558,6 @@ export type XoMirrorBackupJob = BaseXoJob & {
export type XoJob = BaseXoJob & {
type: 'call'
method: string
paramsVector?: {
type: string
items: {
type?: string
values?: {
schedules?: XoSchedule['id'][]
[key: string]: unknown
}[]
[key: string]: unknown
}[]
}
}
export type XoSchedule = {
@@ -614,7 +594,7 @@ export type XoSr = BaseXapiXo & {
VDIs: AnyXoVdi['id'][]
allocationStrategy?: 'thin' | 'thick' | 'unknown'
content_type: 'user' | 'iso' | 'disk'
content_type: string
current_operations: Record<string, STORAGE_OPERATIONS>
id: Branded<'SR'>
inMaintenanceMode: boolean
@@ -625,28 +605,7 @@ export type XoSr = BaseXapiXo & {
shared: boolean
size: number
sm_config: Record<string, string>
// SM drivers shipped with XCP-ng/XenServer
SR_type:
| 'cephfs'
| 'dummy'
| 'ext'
| 'file'
| 'glusterfs'
| 'hba'
| 'iso'
| 'largeblock'
| 'linstor'
| 'lvm'
| 'lvmofcoe'
| 'lvmohba'
| 'lvmoiscsi'
| 'moosefs'
| 'nfs'
| 'shm'
| 'smb'
| 'udev'
| 'xfs'
| 'zfs'
SR_type: string
tags: string[]
type: 'SR'
usage: number
@@ -678,30 +637,13 @@ export type XoTask = {
method?: string
name?: string
objectId?: string
objectType?:
| XapiXoRecord['type']
| 'backup'
| 'backup-archive'
| 'backup-job'
| 'backup-log'
| 'backup-repository'
| 'group'
| 'proxy'
| 'restore'
| 'restore-log'
| 'schedule'
| 'server'
| 'task'
| 'user'
| 'acl-privilege'
| 'acl-role'
params?: Record<string, unknown>
progress?: number
type?: string
userId?: string
[key: string]: unknown | undefined
}
result?: Record<string, unknown>
result: Record<string, unknown>
start: number
status: 'failure' | 'interrupted' | 'pending' | 'success'
tasks?: XoTask[]
@@ -715,7 +657,7 @@ export type XoUser = {
groups: XoGroup['id'][]
id: Branded<'user'>
name?: string
permission: 'none' | 'admin'
permission: string
pw_hash?: string
preferences: Record<string, string>
}
@@ -768,7 +710,6 @@ type BaseXoVdi = BaseXapiXo & {
export type XoVdi = BaseXoVdi & {
id: Branded<'VDI'>
chainPhysicalUsage: number
type: 'VDI'
}
@@ -895,14 +836,14 @@ export type NonXapiXoRecord =
| AnyXoLog
| XoGroup
| XoProxy
| XoGroup
| XoProxy
| XoJob
| XoBackupRepository
| XoSchedule
| XoServer
| XoTask
| XoUser
| XoAclRole
| XoAclBasePrivilege
export type XoRecord = XapiXoRecord | NonXapiXoRecord

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,16 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
}
})

View File

@@ -1,47 +0,0 @@
{
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"main": "dist/index.mjs",
"name": "@xen-orchestra/acl",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
"version": "1.6.1",
"license": "AGPL-3.0-or-later",
"private": false,
"type": "module",
"engines": {
"node": ">= 20.18"
},
"exports": {
".": {
"types": "./dist/index.d.mts",
"default": "./dist/index.mjs"
}
},
"scripts": {
"build": "tsc && node scripts/generate-types.mjs",
"clean": "rimraf dist/",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "npm run build && node --test ./dist/**/*.test.mjs",
"prebuild": "npm run clean",
"prepublishOnly": "npm run build"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"@vates/types": "^2.0.1",
"rimraf": "^6.0.1",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0"
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/acl",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"dependencies": {
"complex-matcher": "^1.0.0"
}
}

View File

@@ -1,55 +0,0 @@
#!/usr/bin/env node
// AUTO-GENERATES dist/generated/privilege-types.d.mts
// Run via `npm run build` (after tsc).
import { mkdirSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../dist/actions/index.mjs'
import { getActionStrings } from '../dist/index.mjs'
function toPascalCase(str) {
return str
.split(/[-_]/)
.map(s => s.charAt(0).toUpperCase() + s.slice(1))
.join('')
}
const resources = Object.keys(SUPPORTED_ACTIONS_BY_RESOURCE)
const typeBlocks = resources.map(resource => {
const actionUnion = getActionStrings(resource)
.map(action => `'${action}'`)
.join(' | ')
return `export type ${toPascalCase(resource)}Privilege = {
id: XoAclBasePrivilege['id']
resource: '${resource}'
action: ${actionUnion}
selector?: XoAclBasePrivilege['selector']
effect: XoAclBasePrivilege['effect']
roleId: XoAclBasePrivilege['roleId']
}`
})
const anyUnion = resources.map(resource => ` | ${toPascalCase(resource)}Privilege`).join('\n')
const types = `\
// -----------------------------------------------------------------------
// AUTO-GENERATED — do not edit manually.
// Regenerated on every \`npm run build\`.
// -----------------------------------------------------------------------
import type { XoAclBasePrivilege } from '@vates/types'
${typeBlocks.join('\n\n')}
export type AnyPrivilege =
${anyUnion}
`
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
const outDir = join(root, 'dist/generated')
mkdirSync(outDir, { recursive: true })
writeFileSync(join(outDir, 'privilege-types.d.mts'), types, 'utf-8')

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
action: true,
effect: true,
resource: true,
selector: true,
},
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
description: true,
groups: true,
name: true,
users: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,13 +0,0 @@
export default {
benchmark: true,
create: true,
read: true,
forget: true,
update: {
enabled: true,
name: true,
options: true,
proxy: true,
url: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,9 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
users: true,
},
}

View File

@@ -1,28 +0,0 @@
export default {
'allow-vm': true,
detach: true,
disable: true,
enable: true,
evacuate: true,
export: {
logs: true,
},
forget: true,
'join-pool': true,
'migrate-receive': true,
read: true,
reboot: {
clean: true,
smart: true,
},
'restart-toolstack': true,
'scan-pifs': true,
shutdown: {
clean: true,
emergency: true,
},
start: true,
update: {
tags: true,
},
}

View File

@@ -1,80 +0,0 @@
import { default as aclPrivilege } from './acl-privilege.mjs'
import { default as aclRole } from './acl-role.mjs'
import { default as alarm } from './alarm.mjs'
import { default as backupArchive } from './backup-archive.mjs'
import { default as backupJob } from './backup-job.mjs'
import { default as backupLog } from './backup-log.mjs'
import { default as backupRepository } from './backup-repository.mjs'
import { default as gpuGroup } from './gpuGroup.mjs'
import { default as group } from './group.mjs'
import { default as host } from './host.mjs'
import { default as message } from './message.mjs'
import { default as network } from './network.mjs'
import { default as pbd } from './pbd.mjs'
import { default as pci } from './pci.mjs'
import { default as pgpu } from './pgpu.mjs'
import { default as pif } from './pif.mjs'
import { default as pool } from './pool.mjs'
import { default as proxy } from './proxy.mjs'
import { default as restoreLog } from './restore-log.mjs'
import { default as schedule } from './schedule.mjs'
import { default as server } from './server.mjs'
import { default as sm } from './sm.mjs'
import { default as sr } from './sr.mjs'
import { default as task } from './task.mjs'
import { default as user } from './user.mjs'
import { default as vbd } from './vbd.mjs'
import { default as vdiSnapshot } from './vdi-snapshot.mjs'
import { default as vdiUnmanaged } from './vdi-unmanaged.mjs'
import { default as vdi } from './vdi.mjs'
import { default as vgpu } from './vgpu.mjs'
import { default as vgpuType } from './vgpuType.mjs'
import { default as vif } from './vif.mjs'
import { default as vmController } from './vm-controller.mjs'
import { default as vmSnapshot } from './vm-snapshot.mjs'
import { default as vmTemplate } from './vm-template.mjs'
import { default as vm } from './vm.mjs'
import { default as vtpm } from './vtpm.mjs'
/**
* value without sub action like 'shutdown' mean: 'shutdown:*' (so shutdown mean you can shutdown:clean and shutdown:hard)
*/
export const SUPPORTED_ACTIONS_BY_RESOURCE = {
'acl-privilege': aclPrivilege,
'acl-role': aclRole,
alarm,
'backup-archive': backupArchive,
'backup-job': backupJob,
'backup-log': backupLog,
'backup-repository': backupRepository,
group,
gpuGroup,
host,
message,
network,
pbd,
pci,
pgpu,
pif,
pool,
proxy,
'restore-log': restoreLog,
schedule,
server,
sm,
sr,
task,
user,
vbd,
'vdi-snapshot': vdiSnapshot,
'vdi-unmanaged': vdiUnmanaged,
vdi,
vgpu,
vgpuType,
vif,
'vm-controller': vmController,
'vm-snapshot': vmSnapshot,
'vm-template': vmTemplate,
vm,
vtpm,
} as const

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,9 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
tags: true,
other_config: true,
},
}

View File

@@ -1,5 +0,0 @@
export default {
read: true,
plug: true,
unplug: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
management: true,
},
}

View File

@@ -1,14 +0,0 @@
export default {
'add-host': true,
create: {
network: true,
vm: true,
},
'emergency-shutdown': true,
read: true,
'rolling-reboot': true,
'rolling-update': true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,4 +0,0 @@
export default {
read: true,
run: true,
}

View File

@@ -1,7 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,16 +0,0 @@
export default {
create: true,
delete: true,
forget: true,
import: {
vdi: true,
vm: true,
},
'migrate-receive': true,
read: true,
'reclaim-space': true,
scan: true,
update: {
tags: true,
},
}

View File

@@ -1,5 +0,0 @@
export default {
abort: true,
delete: true,
read: true,
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
password: true,
permission: true,
preferences: true,
},
}

View File

@@ -1,7 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
}

View File

@@ -1,8 +0,0 @@
export default {
delete: true,
export: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,15 +0,0 @@
export default {
boot: true,
create: true,
delete: true,
'export-content': true,
'import-content': true,
'migrate-send': true,
read: true,
update: {
name_description: true,
name_label: true,
size: true,
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,15 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
update: {
allowedIpv4Addresses: true,
allowedIpv6Addresses: true,
lockingMode: true,
rateLimit: true,
txChecksumming: true,
other_config: true,
},
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
tags: true,
},
}

View File

@@ -1,8 +0,0 @@
export default {
delete: true,
export: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,9 +0,0 @@
export default {
delete: true,
export: true,
instantiate: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,61 +0,0 @@
export default {
clone: true,
delete: true,
export: true,
'migrate-send': true,
pause: true,
read: true,
reboot: {
clean: true,
hard: true,
},
resume: true,
'revert-snapshot': true,
shutdown: {
clean: true,
hard: true,
},
snapshot: true,
start: true,
suspend: true,
unpause: true,
update: {
affinityHost: true,
autoPoweron: true,
blockedOperations: true,
coresPerSocket: true,
cpuCap: true,
cpuMask: true,
cpuWeight: true,
cpus: true,
cpusStaticMax: true,
creation: true,
datasources: true,
expNestedHvm: true,
hasVendorDevice: true,
highAvailability: true,
hvmBootFirmware: true,
memory: true,
memoryMax: true,
memoryMin: true,
memoryStaticMax: true,
nameDescription: true,
nameLabel: true,
nestedVirt: true,
nicType: true,
notes: true,
PV_args: true,
resourceSet: true,
secureBoot: true,
share: true,
startDelay: true,
suspendSr: true,
tags: true,
uefiMode: true,
vga: true,
videoram: true,
viridian: true,
virtualizationMode: true,
xenStoreData: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,119 +0,0 @@
import * as CM from 'complex-matcher'
import type { XoAclPrivilege, XoAclSupportedActions, XoAclSupportedResource } from '@vates/types/lib/xen-orchestra/acl'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../actions/index.mjs'
export type SupportedActionsByResource = typeof SUPPORTED_ACTIONS_BY_RESOURCE
export type SupportedResource = XoAclSupportedResource<SupportedActionsByResource>
export type SupportedActions<T extends SupportedResource> = XoAclSupportedActions<SupportedActionsByResource, T>
export type TPrivilege<T extends SupportedResource> = XoAclPrivilege<SupportedActionsByResource, T>
export class Privilege<T extends SupportedResource> {
#action: TPrivilege<T>['action']
#selector?: (object: object) => boolean
#resource: TPrivilege<T>['resource']
#effect: TPrivilege<T>['effect']
constructor({
action,
selector,
resource,
effect,
}: {
action: TPrivilege<T>['action']
selector?: TPrivilege<T>['selector']
resource: TPrivilege<T>['resource']
effect: TPrivilege<T>['effect']
}) {
Privilege.checkActionIsValid(resource, action)
this.#action = action
this.#selector = selector !== undefined ? CM.parse(selector).createPredicate() : undefined
this.#resource = resource
this.#effect = effect
}
get effect() {
return this.#effect
}
#matchAction<Resource extends SupportedResource = T>(action: SupportedActions<Resource>) {
// read:name_label - read:name_label
if (this.#action === action) {
return true
}
const [namespaceToMatch, actionToMatch] = action.split(':')
const [thisNamespace, thisAction] = this.#action.split(':')
if (thisNamespace === '*') {
return true
}
// update:vm - read:name_label
if (thisNamespace !== namespaceToMatch) {
return false
}
// read - read:name_label
if (thisAction === undefined) {
return true
}
// read:name_decription - read:name_label
if (thisAction !== actionToMatch) {
return false
}
throw new Error(`Unable to verify if ${this.#action} match ${action} `)
}
#matchSelector(object: object) {
if (this.#selector === undefined) {
return true
}
return this.#selector(object)
}
#matchResource(resource: string): resource is SupportedResource {
return resource === this.#resource
}
match<Resource extends SupportedResource = T>(constraint: {
action: SupportedActions<Resource>
resource: Resource
object: object
}): boolean {
return (
this.#matchResource(constraint.resource) &&
this.#matchAction(constraint.action) &&
this.#matchSelector(constraint.object)
)
}
/**
* Throw if action not supported
*/
static checkActionIsValid<T extends SupportedResource>(resource: T, action: SupportedActions<T>) {
const supportedActions = SUPPORTED_ACTIONS_BY_RESOURCE[resource]
if (supportedActions === undefined) {
throw new Error(`${resource} resource not supported`)
}
if (action === '*') {
return
}
const segments = action.split(':')
let _action: object | undefined = undefined
segments.forEach(segment => {
_action = (_action ?? supportedActions)[segment]
if (_action === undefined) {
throw new Error(
`${action} action not supported for the resource: ${resource}. See ${JSON.stringify(supportedActions)}`
)
}
})
}
}

View File

@@ -1,10 +0,0 @@
// DO NOT EDIT
// Overwritten by scripts/generate-types.mjs after running npm run build
// scripts/generate-types.mjs will replace this file to automatically generate and export all privileged types.
// This makes external use easier because it's not necessary to resolve the type every time.
import type { Privilege, SupportedResource } from '../index.mjs'
export type AnyPrivilege = {
[Resource in SupportedResource]: Privilege<Resource>
}[SupportedResource]

View File

@@ -1,167 +0,0 @@
import assert from 'node:assert'
import type { XoUser } from '@vates/types/xo'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
import {
Privilege as CPrivilege,
TPrivilege as Privilege,
SupportedActions,
SupportedActionsByResource,
SupportedResource,
} from './class/privilege.mjs'
// Export all types constructed using SUPPORTED_ACTIONS_BY_RESOURCE
export type { Privilege, SupportedActions, SupportedActionsByResource, SupportedResource }
export { SUPPORTED_ACTIONS_BY_RESOURCE }
export type * from './generated/privilege-types.mjs'
import { AnyPrivilege } from './generated/privilege-types.mjs'
export type AnyPrivilegeOnParam = {
[Resource in SupportedResource]: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: object | object[]
}
}[SupportedResource]
export function hasPrivilegeOn<T extends SupportedResource>({
user,
action,
resource,
objects,
userPrivileges,
}: {
user: XoUser
resource: T
action: SupportedActions<T>
objects: object | object[]
userPrivileges: AnyPrivilege[]
}) {
// Function that will be called outside of the module
// We cannot be sure types are respected
assert.strictEqual(typeof user?.permission, 'string')
assert.strictEqual(typeof action, 'string')
assert.strictEqual(typeof resource, 'string')
assert.strictEqual(objects === undefined, false)
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
if (user.permission === 'admin') {
return true
}
const effectsByObject = new Map<object, Privilege<T>['effect'][]>()
if (Array.isArray(objects)) {
objects.forEach(obj => effectsByObject.set(obj, []))
} else {
effectsByObject.set(objects, [])
}
for (const userPrivilege of userPrivileges) {
const privilege = new CPrivilege(userPrivilege as Privilege<typeof userPrivilege.resource>)
for (const [object, effects] of effectsByObject.entries()) {
if (!privilege.match({ action, resource, object })) {
continue
}
effects.push(privilege.effect)
}
}
if (
Array.from(effectsByObject.values()).some(
effects => effects.length === 0 || effects.some(effect => effect === 'deny')
)
) {
return false
}
return true
}
export function getMissingPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return params
.filter(
param =>
!hasPrivilegeOn({
user: param.user,
resource: param.resource,
action: param.action as SupportedActions<typeof param.resource>,
objects: param.objects,
userPrivileges,
})
)
.map(({ action, objects, resource }) => {
let objectIds: unknown[] | undefined
let objectId: unknown | undefined
if (Array.isArray(objects)) {
if (objects.length > 1) {
objectIds = []
objects.forEach(obj => {
if ('id' in obj) {
objectIds!.push(obj.id)
}
})
} else {
objects = objects?.[0] ?? {}
}
}
if (!Array.isArray(objects)) {
objectId = 'id' in objects ? objects.id : undefined
}
return {
objectId,
objectIds,
action,
resource,
}
})
}
export function hasPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return getMissingPrivileges(params, userPrivileges).length === 0
}
export function filterObjectsWithPrivilege<Resource extends SupportedResource, Object extends object>(param: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: Object[]
userPrivileges: AnyPrivilege[]
}) {
return param.objects.filter(obj => hasPrivilegeOn({ ...param, objects: obj }))
}
// Utils
/**
* Mirror GetKeysRecursively from @vates/types
*/
export function getActionStrings<Resource extends SupportedResource, Actions = SupportedActions<Resource>>(
resource: Resource
): Actions[] {
type NestedActions = { [key: string]: boolean | NestedActions }
function collectActions(obj: NestedActions, prefix?: Actions): Actions[] {
const actions: Actions[] = ['*' as Actions]
for (const [key, value] of Object.entries(obj)) {
const action = (prefix !== undefined ? `${prefix}:${key}` : key) as Actions
actions.push(action)
if (typeof value === 'boolean') {
continue
}
actions.push(...collectActions(value, action))
}
return actions
}
return collectActions(SUPPORTED_ACTIONS_BY_RESOURCE[resource])
}

View File

@@ -1,378 +0,0 @@
import { test, suite } from 'node:test'
import assert from 'node:assert'
import type { Branded } from '@vates/types/common'
import type { XoUser } from '@vates/types/xo'
import { Privilege } from '../class/privilege.mjs'
import type { Privilege as TPrivilege } from '../index.mjs'
import {
filterObjectsWithPrivilege,
getActionStrings,
getMissingPrivileges,
hasPrivilegeOn,
hasPrivileges,
} from '../index.mjs'
suite('Privilege.checkActionIsValid behavior', () => {
test('throw on invalid resource', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('foo', 'read'))
})
test('throw on invalid action', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('vm', 'foo:vm'))
})
test('should work', () => {
assert.doesNotThrow(() => Privilege.checkActionIsValid('vm', 'read'))
})
})
suite('Privilege.match behavior', () => {
const privilege = new Privilege({ action: 'read', resource: 'vm', effect: 'allow' })
const object = { id: 1, name_label: 'foo' }
test('Should not match because of the action', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'start', object }), false)
})
test('Should not match because of the resource', () => {
assert.strictEqual(privilege.match({ resource: 'host', action: 'read', object }), false)
})
test('Should match', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'read', object }), true)
})
test('Should match even if the action is not equal to the action privilege', () => {
const shutdownPrivilege = new Privilege({ action: 'shutdown', resource: 'vm', effect: 'allow' })
assert.strictEqual(shutdownPrivilege.match({ resource: 'vm', action: 'shutdown:clean', object }), true)
})
test('Should match because the action is *', () => {
const vmPrivilege = new Privilege({ action: '*', resource: 'vm', effect: 'allow' })
assert.strictEqual(vmPrivilege.match({ action: 'reboot:hard', resource: 'vm', object: object }), true)
})
test('Should not match due to insufficient privilege', () => {
const cleanShutdownPrivilege = new Privilege({ action: 'shutdown:clean', resource: 'vm', effect: 'allow' })
assert.strictEqual(cleanShutdownPrivilege.match({ resource: 'vm', action: 'shutdown', object }), false)
})
})
suite('ACL V2 behavior', async () => {
// === Setup ===
const admin = {
id: '1' as XoUser['id'],
groups: [],
permission: 'admin',
email: 'admin@admin.net',
preferences: {},
} satisfies XoUser
const user = {
id: '2' as XoUser['id'],
groups: [],
permission: 'none',
email: 'user@user.net',
preferences: {},
} satisfies XoUser
const privilegeId = 'fake-priv-test-id' as Branded<'acl-v2-privilege'>
const roleId = 'fake-role-test-id' as Branded<'acl-v2-role'>
// === Objects
const poolWithoutPrivilege = { id: 'pool-id' }
const alpineVm = { id: 'alpine', creation: { creator: admin.id } }
const almaVm = { id: 'alma', creation: { creator: user.id } }
const xoaVm = { id: 'xoa' }
const template = { id: 'template-id', $pool: '1' }
const templateDeny = { id: 'template-deny-id', $pool: '2' }
const vdis = [{ $SR: '2' }, { $SR: '2' }]
const vdisMultipleSrs = [{ $SR: '1' }, { $SR: '2' }]
const vifs = [{ $network: '1' }]
const iso = { id: 'iso-vdi-id', $SR: '3' }
const hostAffinity = { id: 'affinity-host-id', $pool: '1' }
// === DB Privileges
const vmRead = { id: privilegeId, action: 'read', resource: 'vm', effect: 'allow', roleId } satisfies TPrivilege<'vm'>
const vmCleanShutdown = {
id: privilegeId,
action: 'shutdown:clean',
resource: 'vm',
selector: `creation:creator:${user.id}`,
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm'>
const xoaVmDenyRead = {
id: privilegeId,
action: 'read',
resource: 'vm',
effect: 'deny',
selector: `id: ${xoaVm.id}`,
roleId,
} satisfies TPrivilege<'vm'>
const vdiCreate = {
id: privilegeId,
action: 'create',
resource: 'vdi',
selector: '$SR:2',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const vifCreate = {
id: privilegeId,
action: 'create',
resource: 'vif',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vif'>
const templateInstantiate = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm-template'>
const templateInstantiateDeny = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
selector: '$pool:2',
effect: 'deny',
roleId,
} satisfies TPrivilege<'vm-template'>
const isoVdiUse = {
id: privilegeId,
action: 'boot',
resource: 'vdi',
selector: '$SR:3',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const hostVm = {
id: privilegeId,
action: 'allow-vm',
resource: 'host',
effect: 'allow',
roleId,
} satisfies TPrivilege<'host'>
const allPrivileges = [
vmRead,
vmCleanShutdown,
xoaVmDenyRead,
vdiCreate,
vifCreate,
templateInstantiate,
templateInstantiateDeny,
isoVdiUse,
hostVm,
]
// === Test ===
suite('hasPrivilegeOn behavior', () => {
test('Should always return true for admin', () => {
assert.strictEqual(
hasPrivilegeOn<'pool'>({
user: admin,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
true
)
})
test('Should return false by default for non admin', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
false
)
})
test('Should return true if user has privilege', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: alpineVm, userPrivileges: allPrivileges }),
true
)
})
test('Should return false if user have a both deny and allow privileges', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: xoaVm, userPrivileges: allPrivileges }),
false
)
})
test('Should handle multiple and single object', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm],
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm, xoaVm],
userPrivileges: allPrivileges,
}),
false
)
})
test('Should allow clean shutdown only own VMs', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: almaVm,
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: [almaVm, alpineVm],
userPrivileges: allPrivileges,
}),
false
)
})
})
suite('getMissingPrivileges behavior', () => {
test('Should return empty array if no missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'shutdown:clean', resource: 'vm', objects: almaVm },
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 0)
})
test('Should return only missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege },
{ user, action: '*', resource: 'pool', objects: poolWithoutPrivilege },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 2)
})
test('Should return objectId if only one object in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectId, undefined)
})
test('Should return objectIds if multiple objects in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: [poolWithoutPrivilege, poolWithoutPrivilege] }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectIds, undefined)
})
})
suite('hasPrivileges behavior', () => {
test('Should return false if at least one privilege deny', () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: templateDeny },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
test("Should return false if at least one privilege didn't match", () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdisMultipleSrs },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
})
suite('filterObjectsWithPrivilege behavior', () => {
test('Should return only objects that match privileges', () => {
let vms = [almaVm, alpineVm, xoaVm, xoaVm]
assert.strictEqual(vms.length, 4)
vms = filterObjectsWithPrivilege({
user,
action: 'read',
resource: 'vm',
objects: vms,
userPrivileges: allPrivileges,
})
assert.strictEqual(vms.length, 2)
})
})
})
suite('getActionStrings', () => {
test('returns flat actions for a resource without nesting', () => {
const actions = getActionStrings('host')
assert.ok(actions.includes('read'))
// @ts-ignore A TypeScript error is thrown because `foo` is not of type `SupportedActions<'vm'>`.
// To test for a negative value, the error is ignored.
assert.strictEqual(actions.includes('foo'), false)
})
test('parent action is included alongside its children', () => {
const actions = getActionStrings('vm')
assert.ok(actions.includes('shutdown'))
assert.ok(actions.includes('shutdown:clean'))
assert.ok(actions.includes('shutdown:hard'))
})
})

View File

@@ -1,19 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"module": "nodenext",
"lib": ["ESNext"],
"moduleResolution": "nodenext",
"target": "ES2022",
"skipLibCheck": true,
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
"noImplicitAny": false, // otherwise it will throw an error when importing packages without type definition
"noUncheckedIndexedAccess": true,
"declaration": true
},
"include": ["./src/**/*"],
"exclude": ["./node_modules", "./dist"]
}

View File

@@ -13,11 +13,11 @@ const wrapCall = (fn, arg, thisArg) => {
*
* WARNING: Does not handle plain objects
*
* @template Item, Result, [This=Iterable<Item>]
* @template Item,This
* @param {Iterable<Item>} iterable
* @param {(this: This, item: Item) => (Result | PromiseLike<Result>)} mapFn
* @param {This} [thisArg] defaults to `iterable`
* @returns {Promise<Result[]>}
* @param {(this: This, item: Item) => (Item | PromiseLike<Item>)} mapFn
* @param {This} [thisArg]
* @returns {Promise<Item[]>}
*/
exports.asyncMap = function asyncMap(iterable, mapFn, thisArg = iterable) {
return Promise.all(Array.from(iterable, mapFn, thisArg))
@@ -26,11 +26,11 @@ exports.asyncMap = function asyncMap(iterable, mapFn, thisArg = iterable) {
/**
* Like `asyncMap` but wait for all promises to settle before rejecting
*
* @template Item, Result, [This=Iterable<Item>]
* @template Item,This
* @param {Iterable<Item>} iterable
* @param {(this: This, item: Item) => (Result | PromiseLike<Result>)} mapFn
* @param {This} [thisArg] defaults to `iterable`
* @returns {Promise<Result[]>}
* @param {(this: This, item: Item) => (Item | PromiseLike<Item>)} mapFn
* @param {This} [thisArg]
* @returns {Promise<Item[]>}
*/
exports.asyncMapSettled = function asyncMapSettled(iterable, mapFn, thisArg = iterable) {
return new Promise((resolve, reject) => {

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/async-map",
"version": "0.1.4",
"version": "0.1.2",
"license": "ISC",
"description": "Promise.all + map for all iterables",
"keywords": [
@@ -28,7 +28,7 @@
"node": ">=6"
},
"dependencies": {
"lodash": "^4.18.0"
"lodash": "^4.17.4"
},
"scripts": {
"postversion": "npm publish",

View File

@@ -17,7 +17,7 @@
},
"dependencies": {
"@vates/decorate-with": "^2.1.0",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/log": "^0.7.1",
"golike-defer": "^0.5.1",
"object-hash": "^2.0.1"
},

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,21 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
},
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,56 +0,0 @@
{
"name": "@xen-orchestra/backup-archive",
"version": "1.1.2",
"description": "Helper to handle generic backup archives",
"license": "ISC",
"author": "",
"type": "module",
"main": "./dist/index.mjs",
"exports": {
".": "./dist/index.mjs",
"./disks": "./dist/disks/index.mjs"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^6.0.1",
"@types/node": "^22.3.0",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0",
"fs-extra": "^11.1.0",
"sinon": "^18.0.0",
"tar-stream": "^3.1.8",
"@types/tar-stream": "^3.1.4",
"tmp": "^0.2.1",
"@vates/types": "^2.0.1"
},
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/fs": "^4.9.3",
"@xen-orchestra/disk-transform": "^1.3.3",
"@vates/task": "0.7.0",
"golike-defer": "0.5.1",
"uuid": "^9.0.0",
"promise-toolbox": "^0.21.0",
"@vates/async-each": "^1.0.1",
"vhd-lib": "^4.16.1"
},
"scripts": {
"build": "yarn run clean && tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "yarn run clean && tsc && node --test **/*.test.mjs",
"test-integration": "yarn run clean && tsc && node --test **/*.integ.mjs",
"clean": "rimraf dist/"
},
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/backup-archive",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20.17"
}
}

View File

@@ -1,438 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import {
MergeRemoteDisk,
openDisk,
openDisposableDisk,
RemoteDisk,
isDisk,
openDiskChain,
} from '@xen-orchestra/backup-archive/disks'
import Disposable from 'promise-toolbox/Disposable'
import {
CleanResult,
DEFAULT_MERGE_CONCURRENCY,
DEFAULT_REMOVE_CONCURRENCY,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { asyncEach } from '@vates/async-each'
import { Task } from '@vates/task'
import { limitConcurrency } from 'limit-concurrency-decorator'
import assert from 'node:assert/strict'
const defaultMergeLimiter = limitConcurrency(1)
const MERGE_STATE_FILE_RE = /^\.(.+)\.merge\.json$/
/**
* Tracks the disk chain for a single VDI across all backup snapshots.
* Owns merge and deletion decisions for its chain given which disks are still
* referenced by active backups (accumulated via addActiveDiskPath).
*/
export class RemoteDiskLineage {
#handler: RemoteHandlerAbstract
#vdiDir: string
#opts: ResolvedBackupCleanOptions
// Concrete disk paths discovered in this VDI directory (normalized)
#diskPaths: Set<string> = new Set()
// Disk paths that exist on disk but failed to open (corrupted)
#brokenDiskPaths: Set<string> = new Set()
// child path: parent path (from disk header)
#parentOf: Map<string, string> = new Map()
// parent path: child path
#childOf: Map<string, string> = new Map()
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
#activeDiskPaths: Set<string> = new Set()
// Interrupted merges: normalized parent path { stateFilePath, chain }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
this.#handler = handler
this.#vdiDir = vdiDir
this.#opts = opts
}
#unregisterDisk(diskPath: string): void {
this.#diskPaths.delete(diskPath)
const parent = this.#parentOf.get(diskPath)
if (parent !== undefined) {
this.#parentOf.delete(diskPath)
this.#childOf.delete(parent)
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
this.#childOf.delete(diskPath)
this.#parentOf.delete(child)
}
}
/**
* Lists disk files and reads their headers to build the parent-child chain.
* Also detects interrupted merge state files.
*/
async init(): Promise<void> {
let files: string[]
try {
files = await this.#handler.list(this.#vdiDir, { prependDir: true })
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.#opts.logWarn('failed to list VDI directory', { vdiDir: this.#vdiDir, error })
return
}
for (const filePath of files) {
if (isDisk(filePath)) {
this.#diskPaths.add(normalize(filePath))
}
}
const mergeParentPaths = new Set<string>()
for (const filePath of files) {
const match = MERGE_STATE_FILE_RE.exec(basename(filePath))
if (match !== null) {
mergeParentPaths.add(normalize(this.#vdiDir + '/' + match[1]))
}
}
const uuidToPath = new Map<string, string>()
for (const diskPath of this.#diskPaths) {
let disk: RemoteDisk | undefined
try {
if (mergeParentPaths.has(diskPath)) {
disk = await openDisk({ handler: this.#handler, path: diskPath, force: true })
} else {
disk = await openDisk({ handler: this.#handler, path: diskPath })
}
} catch (error) {
if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
throw error
}
this.#opts.logWarn('failed to open disk', { path: diskPath, error })
// Broken disks will be removed by the clean phase
// because they are marked as NOT active
this.#brokenDiskPaths.add(diskPath)
continue
}
try {
const uuid = disk.getUuid()
// Detect Disks with the same UUIDs
// Due to a bug introduced in a1bcd35e2
const existingPath = uuidToPath.get(uuid)
if (existingPath !== undefined) {
this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
let existingDisk: RemoteDisk | undefined
try {
existingDisk = await openDisk({ handler: this.#handler, path: existingPath })
if (existingDisk.containsAllDataOf(disk)) {
this.#opts.logWarn('should drop duplicate disk, existing is superset', { dropped: diskPath })
continue
} else if (disk.containsAllDataOf(existingDisk)) {
this.#opts.logWarn('should drop old duplicate disk, new is superset', { dropped: existingPath })
} else {
this.#opts.logWarn('duplicate disks have different content, should first-seen', { dropped: diskPath })
continue
}
} catch (error) {
this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
this.#unregisterDisk(existingPath)
} finally {
await existingDisk?.close()
}
}
uuidToPath.set(uuid, diskPath)
if (disk.isDifferencing()) {
const parentPath = disk.getParentPath()
this.#parentOf.set(diskPath, parentPath)
if (this.#childOf.has(parentPath)) {
const error = new Error('this script does not support multiple children', {
cause: {
parent: parentPath,
child1: this.#childOf.get(parentPath),
child2: diskPath,
},
})
throw error
}
this.#childOf.set(parentPath, diskPath)
}
} catch (error) {
this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
} finally {
await disk.close()
}
}
this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
this.#handler,
this.#vdiDir,
files,
uuidToPath,
this.#childOf
)
const propagateBroken = (diskPath: string) => {
if (!this.#brokenDiskPaths.has(diskPath)) {
this.#brokenDiskPaths.add(diskPath)
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
propagateBroken(child)
}
}
}
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent) || this.#brokenDiskPaths.has(parent)) {
this.#opts.logWarn('disk broken: parent missing or broken', { child, parent })
propagateBroken(child)
}
}
}
/**
* Returns true if the disk path exists on disk and opened without errors during init.
* Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
*/
isPathAccessible(diskPath: string): boolean {
return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
}
/** Returns true if the disk exists but failed to open during init. */
isBroken(diskPath: string): boolean {
return this.#brokenDiskPaths.has(diskPath)
}
/** Verifies that every parent referenced in disk headers exists in the directory. */
async check(): Promise<void> {
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent)) {
this.#opts.logWarn('parent disk is missing', { child, parent })
}
}
}
/**
* Declares disk paths as active for this lineage.
* Called by each archive that references this lineage after it determines it is complete.
* Accumulated across all referencing archives before clean() is called.
*/
addActiveDiskPath(diskPath: string): void {
this.#activeDiskPaths.add(normalize(diskPath))
}
/**
* For each orphan disk:
* - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
* - If it has no active descendant → delete it (if opts.remove).
*
* Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
* Deletes orphan merge state files (for missing disks) when opts.remove is true.
*
* Requires activeDiskPaths to be completed to have been called by all referencing archives before this.
* @returns Set of deleted disk paths.
*/
async clean({
remove = this.#opts.remove ?? false,
merge = this.#opts.merge ?? false,
}: { remove?: boolean; merge?: boolean } = {}): Promise<CleanResult> {
const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
const toDelete = new Set<string>()
const toMerge: { chain: string[]; isResuming: boolean }[] = []
const visited = new Set<string>()
// Walk from an orphan toward its descendants.
// Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
// or undefined if the subtree is entirely orphaned (mark for deletion).
const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
if (visited.has(diskPath)) {
return undefined
}
visited.add(diskPath)
if (!orphanDisks.has(diskPath)) {
// Active disk, anchor of the merge chain
return [diskPath]
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
const chain = getUsedChildChainOrDelete(child)
if (chain !== undefined) {
chain.unshift(diskPath)
return chain
}
}
toDelete.add(diskPath)
return undefined
}
// Process interrupted merges first so their disks are protected from the orphan loop
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
if (!this.#diskPaths.has(parentPath)) {
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
if (remove) {
this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
let chain: string[] | undefined
if (stateChain !== undefined) {
const existing = stateChain.filter(p => this.#diskPaths.has(p))
if (existing.length >= 2) chain = existing
}
if (chain !== undefined) {
chain.forEach(p => visited.add(p))
toMerge.push({ chain, isResuming: true })
}
}
for (const orphan of orphanDisks) {
if (!visited.has(orphan)) {
const parent = this.#parentOf.get(orphan)
if (parent === undefined || !orphanDisks.has(parent)) {
const chain = getUsedChildChainOrDelete(orphan)
if (chain !== undefined && chain.length > 1) {
toMerge.push({ chain, isResuming: false })
}
}
}
}
if (remove) {
await asyncEach(
Array.from(toDelete),
async path => {
this.#opts.logInfo('deleting unused disk', { path })
try {
const disk = await openDisk({ handler: this.#handler, path, ignoreBlockIndexes: true })
await disk.unlink({ force: true })
// Remove from tracking so #cleanOrphanDataFiles skips this now-deleted path
this.#unregisterDisk(path)
} catch (error) {
this.#opts.logWarn('failed to delete unused disk', { path, error })
}
},
{ concurrency: DEFAULT_REMOVE_CONCURRENCY }
)
}
if (toMerge.length > 0) {
this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
}
// mergeTargetPath: final size of the disk everything was merged into
const mergedSizes = new Map<string, number>()
if (merge) {
const limitedMergeChain = defaultMergeLimiter(this.#mergeChain.bind(this))
const doMerge = async () => {
await asyncEach(
toMerge,
async ({ chain, isResuming }) => {
const { finalDiskSize, mergeTargetPath } = await limitedMergeChain([...chain], isResuming)
mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
// parentPath alias deleted by parentDisk.rename(mergeTargetPath)
// intermediates deleted by childDisk.unlink() when removeUnused=true
// Unregister so #cleanOrphanDataFiles does not read already-deleted aliases
// and produce false "missing target of alias" warnings.
for (const deletedPath of chain.slice(0, -1)) {
this.#unregisterDisk(deletedPath)
}
},
{ concurrency: this.#opts.mergeConcurrency ?? DEFAULT_MERGE_CONCURRENCY }
)
}
if (toMerge.length !== 0) {
await Task.run({ properties: { name: 'merge' } }, doMerge)
}
}
await this.#cleanOrphanDataFiles(remove)
return {
removedFiles: Array.from(toDelete),
merge: toMerge.length > 0,
size: [...mergedSizes.values()].reduce((total, subSize) => total + subSize, 0),
mergedSizes,
}
}
/**
* Merges ancestors into the active child at the end of the chain.
* Returns the final size of the merge target and its path.
*/
async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
assert.ok(chain.length >= 2, `look to merge a chain shorter than 2 ${JSON.stringify(chain)}`)
const parentPath = chain[0]
this.#opts.logInfo('merging disk chain', { chain: [...chain] }) // need a copy here to log full chain
// The last disk in the chain is the active one that everything gets merged into
const mergeTargetPath = chain.pop()!
const merger = new MergeRemoteDisk(this.#handler, {
logInfo: this.#opts.logInfo,
removeUnused: this.#opts.remove,
mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
onProgress: this.#opts.onProgress,
})
let parentDisk: RemoteDisk | undefined
let childDisk: RemoteDisk | undefined
try {
parentDisk = await openDisk({ handler: this.#handler, path: parentPath, force: isResuming })
childDisk = await openDiskChain({
handler: this.#handler,
path: mergeTargetPath,
until: parentPath,
force: isResuming,
})
const { finalDiskSize } = (await merger.merge(parentDisk, childDisk!))!
return { finalDiskSize, mergeTargetPath }
} finally {
await parentDisk?.close().catch(error => {
this.#opts.logWarn('Error closing parent Disk', error)
})
await childDisk?.close().catch(error => {
this.#opts.logWarn('Error closing child Disk', error)
})
}
}
/**
* Validates each disk integrity, then scans discovered data subdirs
* and deletes any files not associated to a known disk.
*/
async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
const claimedFiles = new Set<string>()
for (const diskPath of this.#diskPaths) {
const claimed = await Disposable.use(
openDisposableDisk({ handler: this.#handler, path: diskPath, ignoreBlockIndexes: true }),
async disk => {
await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
return disk.listAssociatedFiles(this.#vdiDir)
}
)
for (const f of claimed) claimedFiles.add(normalize(f))
}
const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
for (const dataDir of dataDirs) {
const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
for (const item of items) {
if (!claimedFiles.has(normalize(item))) {
this.#opts.logWarn('orphaned data file', { path: item })
if (remove) {
await this.#handler.unlink(item).catch(async err => {
if (err?.code === 'EISDIR') {
await this.#handler.rmtree(item)
} else {
this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
}
})
}
}
}
}
}
}

View File

@@ -1,74 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
export interface PartialBackupMetadata {
mode: 'full' | 'delta'
xva?: string
vhds?: Record<string, string>
vdis?: Record<string, unknown>
isVhdDifferencing?: Record<string, boolean>
size?: number
jobId: string
scheduleId: string
timestamp: number
// 'true' when written by code including the non-NBD qcow2 corruption fix and its force-full
// gate; its absence marks a potentially-corrupt older backup and makes the gate re-probe.
// Preserved (never stamped) across merges here.
// @todo remove this code and the logic associated on 2027 01 01
includeNonNbdQcow2Fix?: boolean
}
export const DEFAULT_MERGE_CONCURRENCY = 1
export const DEFAULT_REMOVE_CONCURRENCY = 4
export interface BackupCleanOptions {
fix?: boolean
merge?: boolean
remove?: boolean
mergeBlockConcurrency?: number
mergeConcurrency?: number
onProgress?: (progress: { total: number; done: number }) => void
logInfo?: (message: any, opts?: object) => void
logWarn?: (message: any, opts?: object) => void
}
export type ResolvedBackupCleanOptions = BackupCleanOptions & {
logInfo: (message: any, opts?: object) => void
logWarn: (message: any, opts?: object) => void
}
export interface ArchiveCleanOptions {
remove?: boolean
merge?: boolean
mergedSizes?: Map<string, number>
}
export interface CheckResult {
isValid: boolean
// Files referenced by metadata but absent on disk
missingDisks?: string[]
// Disk files that exist but failed to open (corrupted)
brokenDisks?: string[]
// Files present on disk but not referenced by metadata
orphans?: string[]
// Files referenced by at least one valid backup
linked?: string[]
}
export type CleanResult = {
removedFiles: string[]
merge?: boolean
size?: number
mergedSizes?: Map<string, number>
}
export interface VmBackupInterface {
handler: RemoteHandlerAbstract
// metadataPath: string
// metadata: PartialBackupMetadata
rootPath: string
opts: BackupCleanOptions
init(): Promise<void>
check(): Promise<CheckResult>
clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
getAssociatedFiles(opts: object): Array<string>
}

View File

@@ -1,289 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, normalize } from '@xen-orchestra/fs/path'
import { resolve } from 'node:path'
import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
import {
ArchiveCleanOptions,
BackupCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
DEFAULT_MERGE_CONCURRENCY,
} from './VmBackup.types.mjs'
import { cleanOrphanDiskDirs } from '@xen-orchestra/backup-archive/disks'
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { promisify } from 'node:util'
import zlib from 'node:zlib'
const gzip = promisify(zlib.gzip)
const gunzip = promisify(zlib.gunzip)
const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
export class VmBackupDirectory implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
files: Array<string> = new Array()
orphans: Set<string> = new Set()
backupArchives: Map<string, VmBackupInterface> = new Map()
opts: ResolvedBackupCleanOptions
// Cached result of the last check() call; invalidated by init()
#checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
#uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
// Set by #checkCacheCount(): the on-disk cache did not match the archives found
// on disk, so clean() must regenerate it even when nothing was merged/removed.
#cacheOutOfSync = false
constructor(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions = {
fix: true,
merge: false,
remove: false,
logInfo,
logWarn,
}
) {
this.handler = handler
this.rootPath = vmBackupPath
this.opts = {
...opts,
fix: opts.fix ?? true,
merge: opts.merge ?? false,
remove: opts.remove ?? false,
logInfo: opts.logInfo ?? logInfo,
logWarn: opts.logWarn ?? logWarn,
}
}
async #readCache(path: string): Promise<Record<string, unknown> | undefined> {
try {
return JSON.parse((await gunzip(await this.handler.readFile(path))).toString())
} catch (error) {
if (error?.code !== 'ENOENT') {
logWarn('failed to read cache', { error, path })
}
}
}
async #writeCache(path: string, data: Record<string, unknown>): Promise<void> {
try {
await this.handler.writeFile(path, await gzip(JSON.stringify(data)), { flags: 'w' })
} catch (error) {
logWarn('failed to write cache', { error, path })
}
}
async init() {
this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
this.#checkResult = undefined
this.#uniqueLineages = new Map()
for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
let metadata: PartialBackupMetadata | undefined = undefined
try {
metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
} catch (error) {
this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
}
if (metadata !== undefined) {
try {
const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
this.backupArchives.set(fullPath, backupArchive)
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
}
}
}
}
getAssociatedFiles({ prefix = false }) {
const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
return prefix ? files : files.map(file => basename(file))
}
async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
await this.#checkCacheCount()
for (const backupArchive of this.backupArchives.values()) {
await backupArchive.check()
}
// allUsedFiles is used for root-level orphan detection only.
// Active disk paths are accumulated per-lineage by each archive during check() above.
const allUsedFiles = new Set<string>([
...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
...this.getAssociatedFiles({ prefix: true }),
])
for (const lineage of this.#uniqueLineages!.values()) {
await lineage.check()
}
const orphans = this.files.filter(file => !allUsedFiles.has(file))
const linked = Array.from(allUsedFiles)
this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
return this.#checkResult
}
async clean({
remove = this.opts.remove ?? false,
merge = this.opts.merge ?? false,
}: ArchiveCleanOptions = {}): Promise<CleanResult> {
// Use cached check result if available, otherwise run check now
const { orphans } = this.#checkResult ?? (await this.check())
let cacheNeedsRegen = false
let someLineageMergedOrShouldBe = false
// Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
const allMergedSizes = new Map<string, number>()
await asyncEach(
Array.from(this.#uniqueLineages!.entries()),
async ([_vdiDir, lineage]) => {
const { mergedSizes, removedFiles, merge: hasPendingMerge } = await lineage.clean({ remove, merge })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
if (hasPendingMerge) {
someLineageMergedOrShouldBe = true
}
for (const [diskPath, size] of mergedSizes || []) {
allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
}
},
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
)
// Delete VDI directories not referenced by any archive
const coveredDirs = new Set(this.#uniqueLineages!.keys())
await cleanOrphanDiskDirs(this.handler, this.rootPath, coveredDirs, {
remove,
logWarn: this.opts.logWarn,
logInfo: this.opts.logInfo,
})
// Let each archive clean its own files (e.g. remove metadata for incomplete backups)
// and update metadata with merged sizes if applicable
await asyncEach(
Array.from(this.backupArchives.values()),
async (archive: VmBackupInterface) => {
const { removedFiles } = await archive.clean({ remove, mergedSizes: allMergedSizes })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
},
{ concurrency: 2 }
)
if (allMergedSizes.size > 0 || cacheNeedsRegen || this.#cacheOutOfSync) {
await this.#regenerateCache()
}
// Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
if (remove) {
await asyncEach(
orphans,
async (orphan: string) => {
try {
await this.handler.unlink(orphan)
} catch (error) {
if (error?.code === 'EISDIR') {
this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
} else if (error?.code !== 'ENOENT') {
throw error
}
}
},
{ concurrency: 2 }
)
}
const size = [...allMergedSizes.values()].reduce((total, merged) => total + merged, 0)
return { removedFiles: orphans, merge: someLineageMergedOrShouldBe, size: size }
}
async #checkCacheCount(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const existingCache = await this.#readCache(cachePath)
const actual = existingCache === undefined ? 0 : Object.keys(existingCache).length
const expected = this.backupArchives.size
this.#cacheOutOfSync = actual !== expected
if (this.#cacheOutOfSync) {
this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
}
}
async #regenerateCache(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const cache: Record<string, object> = {}
for (const [path, archive] of this.backupArchives.entries()) {
const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
cache[path] = { _filename: path, id: path, ...a.metadata }
}
await this.#writeCache(cachePath, cache)
}
async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
let backupArchive: VmBackupInterface
try {
if (metadata.mode === 'full') {
backupArchive = new VmFullBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
resolve('/', this.rootPath, metadata.xva!),
this.opts
)
} else if (metadata.mode === 'delta') {
const rawDiskPaths = metadata.vhds
const diskPaths =
rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
backupArchive = new VmIncrementalBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
diskPaths,
this.opts,
this.#uniqueLineages
)
} else {
throw new Error(`Mode ${metadata.mode} not supported`)
}
} catch (error) {
this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
}
await backupArchive.init()
return backupArchive
}
/**
* Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
* The `lock` option is accepted but ignored (locking is the caller's responsibility).
*/
static async cleanVm(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions & { lock?: boolean } = {}
) {
const { lock: _lock, ...cleanOpts } = opts
const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
await dir.init()
await dir.check()
const { merge, size } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
return { merge, size }
}
}

View File

@@ -1,157 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
import { rimraf } from 'rimraf'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import tar from 'tar-stream'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, vmBackupDir
const vmUuid = 'test-vm-uuid'
const rootPath = `xo-vm-backups/${vmUuid}`
async function createMinimalXva() {
const pack = tar.pack()
pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
pack.finalize()
const chunks = []
for await (const chunk of pack) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
await fs.mkdirp(`${tempDir}/${rootPath}`)
vmBackupDir = new VmBackupDirectory(handler, rootPath)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
async function createFullBackupMetadata(name, xvaName) {
await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
const size = await handler.getSize(`${rootPath}/${xvaName}`)
const metadata = {
mode: 'full',
xva: `./${xvaName}`,
vm: { uuid: vmUuid },
jobId: uniqueId(),
timestamp: Date.now(),
size,
}
await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
return metadata
}
describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
test('init() loads full backup archives from metadata files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await createFullBackupMetadata('backup2.json', 'backup2.xva')
await vmBackupDir.init()
assert.equal(vmBackupDir.backupArchives.size, 2)
})
test('getAssociatedFiles() returns cache files', async () => {
// Create a cache file
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
await vmBackupDir.init()
const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
assert.equal(filesWithPrefix.length, 1)
assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
})
test('check() identifies orphan files', async () => {
// Create a valid backup
await createFullBackupMetadata('backup1.json', 'backup1.xva')
// Create orphan files (not referenced by any metadata)
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
await vmBackupDir.init()
const { orphans, linked } = await vmBackupDir.check()
assert.ok(orphans.some(f => f.includes('orphan.xva')))
assert.ok(orphans.some(f => f.includes('random-file.txt')))
assert.ok(linked.some(f => f.includes('backup1.json')))
assert.ok(linked.some(f => f.includes('backup1.xva')))
})
test('clean() removes orphan files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.json'))
assert.ok(remainingFiles.includes('backup1.xva'))
assert.ok(!remainingFiles.includes('orphan.xva'))
})
test('Missing XVA cleans the metadata', async () => {
await createFullBackupMetadata('metadata.json', 'missing.xva')
await handler.unlink(`${rootPath}/missing.xva`)
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
const remainingFiles = await handler.list(rootPath)
assert.ok(logged.match(/XVA is missing/g))
assert.ok(!remainingFiles.includes('metadata.json'))
})
test('clean() preserves cache.json.gz', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
await VmBackupDirectory.cleanVm(handler, rootPath)
const remainingFiles = await handler.list(rootPath)
assert.equal(remainingFiles.length, 3)
assert.ok(remainingFiles.includes('cache.json.gz'))
})
test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
})
test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
assert.ok(vmBackupDir.backupArchives.get(`/${rootPath}/backup1.json`).isValid)
assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept')
})
})

View File

@@ -1,179 +0,0 @@
import { basename, normalize } from '@xen-orchestra/fs/path'
import assert from 'node:assert'
import { FileDescriptor } from '@xen-orchestra/fs'
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
// https://tools.ietf.org/html/rfc1952.html#page-5
Buffer.from('1F8B', 'hex'),
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
Buffer.from('28B52FFD', 'hex'),
]
const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
return true
}
}
return false
}
const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
if (size <= 1024 || size % 512 !== 0) {
return false
}
const buf = Buffer.allocUnsafe(1024)
assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
return buf.every(byte => byte === 0)
}
export async function isValidXva(
handler: RemoteHandlerAbstract,
filePath: string,
metadataSize?: number
): Promise<boolean> {
// size is longer when encrypted + reading part of an encrypted file is not implemented
if (handler.isEncrypted) {
const size = await handler.getSizeOnDisk(filePath)
return size > 20
}
let isValid = true
const fd = await handler.openFile(filePath, 'r')
try {
const size = await handler.getSize(filePath)
if (size < 20) {
// neither a valid gzip nor tar
return false
}
let validSize = true
if (metadataSize !== undefined) {
validSize = metadataSize === size
}
const validDisk = (await isCompressedFile(handler, fd))
? true // compressed files cannot be validated at this time
: await isValidTar(handler, size, fd)
isValid = validSize && validDisk
} finally {
handler.closeFile(fd).catch(noop)
}
return isValid
}
const noop = (): void => {}
export class VmFullBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
xvaPath: string
isValid?: boolean
missingDisk = false
metadataPath: string
metadata: PartialBackupMetadata
opts: ResolvedBackupCleanOptions
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
opts: ResolvedBackupCleanOptions
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.xvaPath = normalize(xvaPath)
this.opts = opts
}
async init() {}
/**
* As we are not 100% sure if XVA is valid,
* we prefer to warn if it is the case but not remove it.
*/
async check(): Promise<CheckResult> {
if (this.isValid === undefined) {
try {
this.isValid = await isValidXva(this.handler, this.xvaPath, this.metadata.size)
} catch (error) {
this.isValid = false
if (error?.code === 'ENOENT') {
this.missingDisk = true
}
this.opts.logWarn('Issue while checking XVA', { error })
}
if (!this.handler.isEncrypted) {
try {
await this.handler.getSize(`${this.xvaPath}.checksum`)
} catch (error) {
this.opts.logWarn('Checksum file not valid, not blocking', { error })
}
}
}
if (!this.isValid) {
if (this.missingDisk) {
this.opts.logWarn('XVA is missing', { path: this.xvaPath })
} else {
this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
}
}
return { isValid: this.isValid }
}
async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (this.isValid === undefined) {
await this.check()
}
const removedFiles: string[] = []
if (!this.isValid && this.missingDisk) {
removedFiles.push(this.metadataPath, `${this.xvaPath}.checksum`)
}
if (remove) {
if (!this.isValid) {
if (!this.missingDisk) {
this.opts.logWarn(`This files may be corrupted but not yet to be removed`, {
files: this.getAssociatedFiles({ prefix: false }),
})
} else {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
}
}
return { removedFiles, merge: false }
}
getAssociatedFiles({ prefix = false }): Array<string> {
let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
return prefix ? validFiles : validFiles.map(file => basename(file))
}
}

View File

@@ -1,859 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import { RemoteVhdDisk } from '../dist/disks/RemoteVhdDisk.mjs'
import { MergeRemoteDisk } from '../dist/disks/MergeRemoteDisk.mjs'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
import { dirname, basename } from 'node:path'
import { promisify } from 'node:util'
import zlib from 'node:zlib'
import { rimraf } from 'rimraf'
const { beforeEach, afterEach, describe } = test
const gunzip = promisify(zlib.gunzip)
let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
const rootPath = 'xo-vm-backups/VMUUID/'
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
jobId = uniqueId()
vdiId = uniqueId()
relativePath = `vdis/${jobId}/${vdiId}`
basePath = `${rootPath}/${relativePath}`
vdiId2 = uniqueId()
relativePath2 = `vdis/${jobId}/${vdiId2}`
basePath2 = `${rootPath}/${relativePath2}`
await fs.mkdirp(`${tempDir}/${basePath}`)
await fs.mkdirp(`${tempDir}/${basePath2}`)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
async function generateVhd(path, opts = {}) {
let vhd
let dataPath = path
if (opts.useAlias) {
await handler.mkdir(dirname(path) + '/data/')
dataPath = dirname(path) + '/data/' + basename(path)
}
if (opts.mode === 'directory') {
await handler.mkdir(dataPath)
vhd = new VhdDirectory(handler, dataPath)
} else {
const fd = await handler.openFile(dataPath, 'wx')
vhd = new VhdFile(handler, fd)
}
vhd.header = { ...VHDHEADER, ...opts.header }
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
if (vhd.header.parentUuid) {
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
if (opts.useAlias === true) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
if (opts.blocks) {
for (const blockId of opts.blocks) {
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
}
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
return vhd
}
test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
// metadata references a non-existent disk so the lineage is created for this vdiDir
// and notReallyAVhd.vhd is treated as an unreferenced orphan
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
assert.ok(logged.includes('failed to open disk'))
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent, should be deleted
await generateVhd(`${basePath}/abandoned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// one orphan, which is a full vhd, no parent : should stay
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan in the metadata : should stay
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
})
)
// clean
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
// abandoned has a missing parent, orphan and child have no active backup reference
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
assert.equal(detectedMissingParent.length, 1)
// we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
})
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.json is not here
],
})
)
await generateVhd(`${basePath}/abandoned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
let matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.vhd is not here anymore
],
}),
{ flags: 'w' }
)
logged = ''
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after metadata deletion'
)
})
test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
// A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
// Metadata references the corrupt VHD as if it were a valid active disk
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/broken.vhd`],
})
)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
logInfo: () => {},
logWarn: logInfo,
})
assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
// metadata removed because backup is incomplete
assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
// broken VHD removed via the orphan path in lineage.clean()
assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
})
test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
// Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
// Each snapshot is the active disk for exactly one backup metadata file.
//
// Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
// own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
// RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
// their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
// deleted — destroying two live backups.
const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
})
await generateVhd(`${basePath}/snapshot3.vhd`, {
header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
})
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata3.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
assert.equal(remaining.length, 3, 'no extra files should remain')
// All three metadata files must also survive (all backups are complete)
const rootFiles = await handler.list(rootPath)
assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
})
test('it handles multiple VDI directories independently', async () => {
// vdiDir1: diskA1 (base) <- diskA2, plus an orphan
const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
await generateVhd(`${basePath}/diskA2.vhd`, {
header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
})
await generateVhd(`${basePath}/orphanA.vhd`)
// vdiDir2: diskB1 (base) <- diskB2, plus an orphan
const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
await generateVhd(`${basePath2}/diskB2.vhd`, {
header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
})
await generateVhd(`${basePath2}/orphanB.vhd`)
// archive1 covers both base disks, archive2 covers both child disks
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining1 = await handler.list(basePath)
assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
const remaining2 = await handler.list(basePath2)
assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
})
test('it merges delta of non destroyed chain', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000, // a size too small
vhds: [
`${relativePath}/grandchild.vhd`, // grand child should not be merged
`${relativePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
blocks: [0, 1],
})
// a grand child
await generateVhd(`${basePath}/grandchild.vhd`, {
header: {
parentUnicodeName: 'child.vhd',
parentUuid: child.footer.uuid,
},
})
let logged = []
const logInfo = message => {
logged.push(message)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
assert.equal(logged[0], `unexpected number of entries in backup cache`)
logged = []
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
assert.equal(logged.includes(`merging disk chain`), true)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 2)
assert.equal(remainingVhds.includes('child.vhd'), true)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
assert.equal(metadata.size, 4299776, 'metadata.size should be updated after merge')
})
test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
// ancestor -> child -> grandchild
// only grandchild is referenced in metadata: both ancestor and child are orphans
// they must be merged as a single 3-disk chain, not processed separately
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000,
vhds: [`${relativePath}/grandchild.vhd`],
})
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
const child = await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
})
await generateVhd(`${basePath}/grandchild.vhd`, {
header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
})
const mergeChains = []
const logInfo = (message, opts) => {
if (message === 'merging disk chain') mergeChains.push(opts.chain)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
// all three disks must be handled in exactly one merge, not two separate operations
assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
assert.equal(remainingVhds.includes('child.vhd'), false)
})
test('it resumes, through cleanVm, an interrupted merge whose ancestor lost its footer', async () => {
// Reproduces the production "footer1 !== footer2" failure end-to-end via the full cleanVm
// pipeline. ancestor (full) <- child (differencing, the active backup).
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [2, 3, 4, 5],
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', size: 12000, vhds: [`${relativePath}/child.vhd`] })
)
// Simulate a merge killed mid-transfer: write 2 of the 4 child blocks into the ancestor
// (which overwrites its end footer) then throw before mergeMetadata() rewrites the footer.
// This leaves the ancestor footerless AND a `.merge.json` state file — the exact post-crash
// state that makes cleanVm take the resume path.
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
await parent.init({ force: false })
await child.init({ force: false })
const merger = new MergeRemoteDisk(handler, { removeUnused: true, writeStateDelay: 0 })
let written = 0
const realWriteBlock = parent.writeBlock.bind(parent)
parent.writeBlock = async diskBlock => {
if (written === 2) {
throw new Error('simulated interruption')
}
written++
return realWriteBlock(diskBlock)
}
await assert.rejects(merger.merge(parent, child), /simulated interruption/)
assert.ok(
(await handler.list(basePath)).includes('.ancestor.vhd.merge.json'),
'an interrupted-merge state file should remain'
)
// Full pipeline: cleanVm must recognize the interrupted merge and resume it — opening the
// footerless ancestor through the chain without throwing "footer1 !== footer2".
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
merge: true,
logInfo: () => {},
logWarn: () => {},
})
// The merge completed: the ancestor was consolidated into child.vhd and the state file is gone.
const remaining = await handler.list(basePath)
assert.equal(remaining.includes('.ancestor.vhd.merge.json'), false, 'state file removed after successful resume')
assert.equal(remaining.includes('ancestor.vhd'), false, 'ancestor consolidated into the merge target')
assert.equal(remaining.includes('child.vhd'), true, 'merge target remains')
// No blocks lost: the resumed merge wrote the blocks (4, 5) that were never written before
// the crash, so the consolidated disk holds the full set.
const merged = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
await merged.init({ force: false })
try {
assert.deepEqual(merged.getBlockIndexes(), [0, 1, 2, 3, 4, 5], 'all blocks present after resumed merge')
} finally {
await merged.close()
}
// The backup is consistent, so its metadata is preserved.
assert.ok((await handler.list(rootPath)).includes('metadata.json'), 'metadata should be preserved')
})
test('it does not warn "missing target of alias" for aliases deleted by merge', async () => {
// Regression: RemoteDiskLineage.#cleanOrphanDataFiles iterated #diskPaths entries just
// deleted during the same clean() call's merge phase, producing false "missing target of
// alias" warnings
const orphan = await generateVhd(`${basePath}/orphan.vhd`, {
useAlias: true,
blocks: [0],
})
await generateVhd(`${basePath}/child.vhd`, {
useAlias: true,
header: {
parentUnicodeName: 'orphan.vhd.alias.vhd',
parentUuid: orphan.footer.uuid,
},
blocks: [1],
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd.alias.vhd`],
})
)
const warnings = []
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
merge: true,
logWarn: (message, data) => warnings.push({ message, data }),
logInfo: () => {},
})
const missingTargetWarnings = warnings.filter(w => w.message === 'missing target of alias')
assert.equal(
missingTargetWarnings.length,
0,
`"missing target of alias" must not fire after merge; got: ${JSON.stringify(missingTargetWarnings)}`
)
const remaining = await handler.list(basePath)
assert.ok(!remaining.includes('orphan.vhd.alias.vhd'), 'orphan alias must be deleted after merge')
assert.ok(remaining.includes('child.vhd.alias.vhd'), 'child alias must survive as merge target')
})
test('it resumes an interrupted merge with chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [5, 6],
})
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
blocks: [7, 8],
})
await handler.writeFile(
`${basePath}/.ancestor.vhd.merge.json`,
JSON.stringify({
parent: { uuid: '0' },
child: { uuid: '0' },
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
currentBlock: 3,
mergedDataSize: 3 * 2 * 1024 * 1024,
step: 'mergeBlocks',
diskSize: 0,
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const files = await handler.list(basePath)
assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
})
test('it resumes an interrupted merge without chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 209920,
vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
// a merge in progress file
await handler.writeFile(
`${basePath}/.orphan.vhd.merge.json`,
JSON.stringify({
parent: {
header: orphan.header.checksum,
},
child: {
header: child.header.checksum,
},
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 1)
assert.equal(remainingVhds.includes('child.vhd'), true)
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
describe('tests multiple combination ', { concurrency: 1 }, () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
// a broken VHD
if (useAlias) {
await handler.mkdir(basePath + '/data')
}
const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
if (vhdMode === 'directory') {
await handler.mkdir(brokenVhdDataPath)
} else {
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
}
if (useAlias) {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
await generateVhd(`${basePath}/abandoned.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// an ancestor of a vhd present in metadata
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
blocks: [1, 3],
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: ancestor.footer.uuid,
},
blocks: [1, 2],
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: child.footer.uuid,
},
blocks: [2, 3],
})
// an older parent that was merging in clean
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
useAlias,
mode: vhdMode,
})
// a clean vhd in metadata
const clean = await generateVhd(`${basePath}/clean.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: cleanAncestor.footer.uuid,
},
})
await handler.writeFile(
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
JSON.stringify({
parent: {
header: cleanAncestor.header.checksum,
},
child: {
header: clean.header.checksum,
},
})
)
// the metadata file
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
`${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
`${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
],
})
)
if (!useAlias && vhdMode === 'directory') {
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
} catch (err) {
assert.strictEqual(
err.code,
'NOT_SUPPORTED',
'Merging directory without alias should raise a not supported error'
)
return
}
assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 6501888)
// cache.json.gz must be regenerated when metadata changes after a merge
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after a merge'
)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
if (useAlias) {
const dataSurvivors = await handler.list(basePath + '/data')
// the goal of the alias : do not move a full folder
assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
assert.equal(survivors.length, 4) // the 3 ok + data
assert.equal(dataSurvivors.length, 3)
} else {
assert.equal(survivors.includes('clean.vhd'), true)
assert.equal(survivors.includes('child.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd'), true)
assert.equal(survivors.length, 3)
}
})
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
// metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('check all types of aliases, corrupted, missing or not', async () => {
// valid alias pointing to an existing data file
await handler.mkdir(`${basePath}/data`)
await generateVhd(`${basePath}/data/ok.vhd`)
await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
// broken alias pointing to a missing data file
await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
// alias pointing to an existing but corrupt data file
await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
// data file without alias
await generateVhd(`${basePath}/data/missingalias.vhd`)
// only the valid alias is referenced by a complete backup
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
// only ok.alias.vhd has survived
const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
assert.equal(aliases.length, 1)
assert.equal(aliases[0], 'ok.alias.vhd')
// corrupt.vhd and missingalias.vhd must be gone
const data = await handler.list(`${basePath}/data`)
assert.equal(data.length, 1)
assert.equal(data[0], 'ok.vhd')
})
test('it preserves VDI directory when handler.list() throws during lineage init', async () => {
// Regression: when handler.list(vdiDir) threw (e.g. S3 404), the lineage
// was never added to #uniqueLineages, so cleanOrphanDiskDirs treated the directory
// as an orphan and deleted it with rmtree()
await generateVhd(`${basePath}/snapshot.vhd`)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot.vhd`] })
)
const originalList = handler.list.bind(handler)
handler.list = async (path, opts) => {
if (typeof path === 'string' && path.includes(vdiId)) {
throw Object.assign(new Error('Simulated transient storage error'), { code: 'ENOENT' })
}
return originalList(path, opts)
}
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
} finally {
handler.list = originalList
}
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot.vhd'), 'VHD must survive when handler.list() throws during lineage init')
})
test('it regenerates the cache when its entry count is out of sync, even with nothing to merge/remove', async () => {
// a referenced disk + its metadata, but no cache.json.gz on disk
await generateVhd(`${basePath}/diskA1.vhd`)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`] })
)
await assert.rejects(handler.readFile(`${rootPath}/cache.json.gz`), { code: 'ENOENT' })
const logged = []
// remove/merge both false: cacheNeedsRegen stays false and no merge happens,
// so only the cache-count mismatch can trigger the regeneration
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: false,
merge: false,
logInfo: () => {},
logWarn: message => logged.push(message),
})
assert.ok(logged.includes('unexpected number of entries in backup cache'), 'mismatch should be reported')
const cache = JSON.parse((await gunzip(await handler.readFile(`${rootPath}/cache.json.gz`))).toString())
assert.equal(Object.keys(cache).length, 1, 'cache should be regenerated to match the archives on disk')
})

Some files were not shown because too many files have changed in this diff Show More