Compare commits

..

3 Commits

Author SHA1 Message Date
Thierry
5dba45f6cf update readme and componentstory to use new menu components 2025-01-10 10:38:40 +01:00
Thierry
b112a72f23 continue menu implementation 2025-01-10 10:08:36 +01:00
Thierry
069ed9215f feat(web-stack): start new menu implementation 2025-01-09 11:32:52 +01:00
2134 changed files with 43704 additions and 129348 deletions

View File

@@ -171,6 +171,7 @@ module.exports = {
'vue/no-empty-component-block': 'error',
'vue/no-multiple-objects-in-class': 'error',
'vue/no-ref-object-reactivity-loss': 'error',
'vue/no-required-prop-with-default': 'error',
'vue/no-static-inline-styles': 'error',
'vue/no-template-target-blank': 'error',
'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
@@ -191,14 +192,18 @@ module.exports = {
// Specific rules for XO i18n-related files
{
files: [
'@xen-orchestra/lite/**/*.{vue,ts}',
'@xen-orchestra/web/**/*.{vue,ts}',
'@xen-orchestra/lite/**/*.{vue,ts,json}',
'@xen-orchestra/web/**/*.{vue,ts,json}',
'@xen-orchestra/web-core/**/*.{vue,ts,json}',
],
extends: ['plugin:@intlify/vue-i18n/base'],
settings: {
'vue-i18n': {
localeDir: ['@xen-orchestra/web-core/lib/locales/*.json'],
localeDir: [
'@xen-orchestra/lite/src/locales/*.json',
'@xen-orchestra/web/src/locales/*.json',
'@xen-orchestra/web-core/lib/locales/*.json',
],
messageSyntaxVersion: '^9.9.0',
},
},
@@ -216,45 +221,13 @@ module.exports = {
'@intlify/vue-i18n/no-raw-text': [
'error',
{
ignoreText: [
'Xen Orchestra Lite',
'Xen Orchestra',
'Xen Orchestra Appliance',
'XO Lite',
'XCP-ng',
'XO 5',
'404',
'⚠️',
],
ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
},
],
'@intlify/vue-i18n/no-v-html': 'error',
'@intlify/vue-i18n/valid-message-syntax': 'error',
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
'@intlify/vue-i18n/no-missing-keys-in-other-locales': [
'error',
{
ignoreLocales: [
'de',
'fa',
'cs',
'es',
'sv',
'it',
'ru',
'uk',
'nl',
'pt-BR',
'da',
'ko',
'nb-NO',
'ja',
'fi',
'pl',
'zh-Hans',
],
},
],
'@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa', 'cs'] }],
},
},
// Specific rules for XO dev-related files
@@ -288,17 +261,6 @@ module.exports = {
'eslint-comments/no-unlimited-disable': 'off',
},
},
{
files: ['*.d.ts'],
parser: '@typescript-eslint/parser',
parserOptions: {
sourceType: 'module',
},
rules: {
'n/no-unsupported-features/es-syntax': 'off',
'no-unused-vars': 'off',
},
},
],
ignorePatterns: ['@vates/fatfs/'],

View File

@@ -40,8 +40,8 @@ body:
- type: markdown
attributes:
value: |
- Don't forget to [read this first](https://docs.xen-orchestra.com/community)
- As well as follow [this guide](https://docs.xen-orchestra.com/community#report-a-bug)
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
- type: input
id: xo-sources-commit-number
attributes:

View File

@@ -1,4 +1,4 @@
name: Continuous Integration
name: Continous Integration
on: push
jobs:
CI:

2
.gitignore vendored
View File

@@ -7,8 +7,6 @@
/@vates/*/node_modules/
/@xen-orchestra/*/dist/
/@xen-orchestra/*/node_modules/
/@xen-orchestra/rest-api/open-api/spec/swagger.json
/@xen-orchestra/rest-api/src/open-api/routes/routes.ts
/packages/*/dist/
/packages/*/node_modules/

View File

@@ -16,7 +16,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
- `signal`: an abort signal to stop the iteration
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'

View File

@@ -34,7 +34,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
- `signal`: an abort signal to stop the iteration
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'

View File

@@ -1,5 +0,0 @@
export function asyncEach<Item>(
iterable: Iterable<Item> | AsyncIterable<Item>,
iteratee: (item: Item, index: number, iterable: Iterable<Item> | AsyncIterable<Item>) => Promise<void>,
opts?: { concurrency?: number; signal?: AbortSignal; stopOnError?: boolean }
): Promise<void>

View File

@@ -11,8 +11,8 @@ class AggregateError extends Error {
/**
* @template Item
* @param {Iterable<Item>|AsyncIterable<Item>} iterable
* @param {(item: Item, index: number, iterable: Iterable<Item>|AsyncIterable<Item>) => Promise<void>} iteratee
* @param {Iterable<Item>} iterable
* @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
* @returns {Promise<void>}
*/
exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
@@ -73,20 +73,20 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
}
nextIsRunning = true
if (running < concurrency) {
try {
const cursor = await it.next()
if (cursor.done) {
next = () => {
if (running === 0) {
if (errors.length === 0) {
resolve()
} else {
reject(new AggregateError(errors))
}
const cursor = await it.next()
if (cursor.done) {
next = () => {
if (running === 0) {
if (errors.length === 0) {
resolve()
} else {
reject(new AggregateError(errors))
}
}
} else {
++running
}
} else {
++running
try {
const result = iteratee.call(this, cursor.value, index++, iterable)
let then
if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
@@ -94,9 +94,9 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
} else {
onFulfilled(result)
}
} catch (error) {
onRejected(error)
}
} catch (error) {
onRejected(error)
}
nextIsRunning = false
return next()

View File

@@ -114,20 +114,6 @@ describe('asyncEach', () => {
tracker.verify()
})
it('handle error on iterable', async () => {
const iteratee = spy(async () => {})
const iterable = async function* () {
for (const value of [1, 2, 3]) {
if (value === 2) {
throw new Error('iterator error')
}
await randomDelay()
yield value
}
}
await assert.rejects(() => asyncEach(iterable(), iteratee, { concurrency: 1 }), { message: 'iterator error' })
})
})
)
})

View File

@@ -24,8 +24,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"types": "index.d.ts",
"version": "1.0.1",
"version": "1.0.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -7,7 +7,7 @@
* @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
* @param {T} data2 - The second data array or buffer to compare.
* @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided; otherwise, undefined.
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
*/
module.exports = function diff(data1, data2, cb) {
let result

View File

@@ -14,7 +14,7 @@ import { deduped } from '@vates/disposable/deduped'
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
const getConnection = deduped(async function (host)) {
const connection = new Connection(host)
return new Disposable(connection, () => connection.close())
return new Disposabe(connection, () => connection.close())
}, host => [host])
```

View File

@@ -32,7 +32,7 @@ import { deduped } from '@vates/disposable/deduped'
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
const getConnection = deduped(async function (host)) {
const connection = new Connection(host)
return new Disposable(connection, () => connection.close())
return new Disposabe(connection, () => connection.close())
}, host => [host])
```

3
@vates/fatfs/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
.DS_Store
.vscode/
node_modules/

View File

@@ -28,7 +28,7 @@ fs.stat('autoexec.bat', function (e, stats) {
## API
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back an instance of an [`fs`](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back the API documented [here](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
- `'ready'` event — fired on `fs` when initial volume information has been determined and the API is ready to use. It is safe to call other `fs` methods before this fires **only if** you are sure the first sector will be readable and represents a valid FAT volume.
- `'error'` event — fired if initialization fails for whatever reason.
@@ -36,7 +36,7 @@ fs.stat('autoexec.bat', function (e, stats) {
The `opts` dictionary you pass to `fatfs.createFileSystem` can contain any of the following options:
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overridden to `true`.
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overriden to `true`.
- `noatime` — The FAT filesystem can track the last access time (just a date, actually) but this means every read operation would also incur some write overhead. Defaults to `true`, meaning by default access times will **not** be updated on reads. Set this to `false` to track access times.
- `modmode` — chooses how `fs.chmod` (and the mode field from `fs.stat`–family calls) should map FAT attributes to POSIX permissions. Set to the number `0111` to map the readonly flag to the user's write bit being unset, and the archive/system/hidden flags to the user/group/other executable bits respectively. Set to the number `07000` to map the readonly flag to _all_ write bits being unset, and the archive/system/hidden flags to the sticky/setgid/setuid bits respectively. Set to `null` for readonly mapping. Defaults to `0111`.
- `umask` — any bits _set_ in this octal number will be _unset_ in the 'mode' field from `fs.stat`–family calls. It does not affect anything else. Defaults to `process.umask()`, or `0022` if that is unavailable.
@@ -53,7 +53,7 @@ Well, sort of…
### Temporary
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please, please, please, **make sure you have a backup** of any important drive/image/card you unleash this upon.
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please please please **make sure you have a backup** of any important drive/image/card you unleash this upon.
- A few methods are not quite implemented, either: `fs.rename`, `fs.unlink` and `fs.rmdir`, as well as `fs.watchFile`/`fs.unwatchFile` and `fs.watch`. These are Coming Soon™.
- There are several internal housekeeping items (redundant FAT tables, extra FAT32 information, etc.) that are not done. These do not seem to affect interop, but you may see warnings when repairing a filesystem written by this module.
- Oh, and not to scare you, but if an IO error happens while writing, the library usually just bails — bubbling an error up to your callback as if it were a hot potato. Although some attempt has been made to do separate writes in the safest order (e.g. allocating an additional file cluster, then appending data into it, and then finally updating the file's size), but this behavior has not been thoroughly audited for all operations. There's certainly no attempt to retry/cleanup/rollback if a multi-step change runs into trouble partway through.

View File

@@ -62,7 +62,7 @@ exports.parseFlags = function (flags) {
case 'ax': info = {read:false, write:true, create:true, append:true, exclusive:true}; break;
case 'a+': info = {read:true, write:true, create:true, append:true}; break;
case 'ax+': info = {read:true, write:true, create:true, append:true, exclusive:true}; break;
default: throw Error("Unknown mode: "+flags); // TODO: throw as `S.err.INVAL`
default: throw Error("Uknown mode: "+flags); // TODO: throw as `S.err.INVAL`
}
if (info.sync) throw Error("Mode not implemented."); // TODO: what would this require of us?
if (_dir) info._openDir = true;

View File

@@ -1,16 +1,15 @@
{
"private": false,
"name": "@vates/fatfs",
"version": "0.11.1",
"version": "0.11.0",
"description": "fs implementation on top of raw FAT16/FAT32 block source",
"main": "index.js",
"scripts": {
"postversion": "npm publish --access public"
},
"repository": {
"directory": "@vates/fatfs",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
"url": "https://github.com/vatesfr/xen-orchestra"
},
"keywords": [
"filesystem",
@@ -21,14 +20,13 @@
],
"author": "Vates SAS",
"license": "Apache-2.0",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fatfs",
"bugs": {
"url": "https://github.com/vatesfr/xen-orchestra/issues"
},
"homepage": "https://github.com/vatesfr/xen-orchestra",
"dependencies": {
"fifolock": "^1.0.0",
"struct-fu": "^1.2.1",
"xok": "^1.0.0"
},
"engines": {
"node": ">=18"
}
}

View File

@@ -90,7 +90,7 @@ function startTests(vol, waitTime) {
assert(path === file, "Fixed fluffy path matches normal one.");
});
fs.realpath([BASE_DIR,"non","existent","path"].join('/'), function (e) {
assert(e, "Expected error calling fs.realpath on nonexistent file.");
assert(e, "Expected error calling fs.realpath on non-existent file.");
});
fs.readdir(BASE_DIR, function (e, arr) {

View File

@@ -13,7 +13,7 @@ await mount(handler, diskId, mountPoint)
### cli
From the install folder:
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
@@ -25,25 +25,4 @@ After installing the package
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
The `remoteUrl` can be found using the following command from the XOA CLI, and then picking the right `url` property:
`xo-server-db ls remote`
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
## Restore a file from a VHD using `fuse-vhd` CLI
1. Mount a VHD to the Filesystem (see **Usage**).
2. Verify the VHD is Correctly Mounted:
`mount | grep fuse`
3. Get the `START` and `SIZE` of the disk whose file you want to restore. Multiply `START` by 512 (block size) to get the offset value. (`mountedVhdPath` is the `mountPoint` value from earlier, followed by `/vhd0`, for example). Depending on the partition type, you may need to do some additional setup to discover the partitions.
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`
5. Copy your files from the mounted partition, then unmount the partition.
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`

View File

@@ -29,7 +29,7 @@ await mount(handler, diskId, mountPoint)
### cli
From the install folder:
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
@@ -41,28 +41,7 @@ After installing the package
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
The `remoteUrl` can be found using the following command from the XOA CLI, and then picking the right `url` property:
`xo-server-db ls remote`
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
## Restore a file from a VHD using `fuse-vhd` CLI
1. Mount a VHD to the Filesystem (see **Usage**).
2. Verify the VHD is Correctly Mounted:
`mount | grep fuse`
3. Get the `START` and `SIZE` of the disk whose file you want to restore. Multiply `START` by 512 (block size) to get the offset value. (`mountedVhdPath` is the `mountPoint` value from earlier, followed by `/vhd0`, for example). Depending on the partition type, you may need to do some additional setup to discover the partitions.
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`
5. Copy your files from the mounted partition, then unmount the partition.
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
## Contributions

View File

@@ -19,12 +19,12 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.6.7",
"@xen-orchestra/fs": "^4.3.0",
"@xen-orchestra/log": "^0.7.1",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.14.7"
"vhd-lib": "^4.11.2"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"

View File

@@ -1,58 +0,0 @@
A toolbox to ease the use of generator
### Timeout
wrap a source async generator to have it throw an error when timeout is reached
timeout is a positive number in milliseconds
```js
import { Timeout } from '@vates/generator-toolbox'
const wrappedGenerator = new Timeout(sourceGenerator, timeout)
```
### Throttle
wrap a source async generator to have it respect a max speed ( in bytes per seconds).
speed is either a strictly positive number or a function returning a strictly positive number. A speed change will be used for the next emitted packet.
The source generator must yield object with a length property.
Optimized for small yields regarding to the speed, since it won't split incoming packet.
If the generator reached the max speed it will be paused, limiting memory consumption.
```js
import { Throttle } from '@vates/generator-toolbox'
const wrappedGenerator = new Throttle(sourceGenerator, speed)
```
### Synchronized
Fork a generator. The rules are:
- if the source returns, all the forks return
- if the forks error, all the forks error with the same error
- if a fork returns, it is stopped, but the generator continue with the other
- if a fork errors, it is stopped, but the generator continue with the other
- if all the forks return, the source is stopped
- if all the forks error, the source is errored with the last error
- the source start producing a packet when the fastest forked ask for it
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
```ts
import { Synchronized } from '@vates/generator-toolbox'
async function consume(generator: AsyncGenerator) {
for await (const val of generator) {
console.log({ val })
}
}
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await Promise.all([consume(first), consume(second)])
```
Note: you can stop early a generator by calling `generator.return()`, and you can stop in in error by calling `generator.throw(error)`

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,89 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/generator-toolbox
[![Package Version](https://badgen.net/npm/v/@vates/generator-toolbox)](https://npmjs.org/package/@vates/generator-toolbox) ![License](https://badgen.net/npm/license/@vates/generator-toolbox) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/generator-toolbox)](https://bundlephobia.com/result?p=@vates/generator-toolbox) [![Node compatibility](https://badgen.net/npm/node/@vates/generator-toolbox)](https://npmjs.org/package/@vates/generator-toolbox)
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/generator-toolbox):
```sh
npm install --save @vates/generator-toolbox
```
## Usage
A toolbox to ease the use of generator
### Timeout
wrap a source async generator to have it throw an error when timeout is reached
timeout is a positive number in milliseconds
```js
import { Timeout } from '@vates/generator-toolbox'
const wrappedGenerator = new Timeout(sourceGenerator, timeout)
```
### Throttle
wrap a source async generator to have it respect a max speed ( in bytes per seconds).
speed is either a strictly positive number or a function returning a strictly positive number. A speed change will be used for the next emitted packet.
The source generator must yield object with a length property.
Optimized for small yields regarding to the speed, since it won't split incoming packet.
If the generator reached the max speed it will be paused, limiting memory consumption.
```js
import { Throttle } from '@vates/generator-toolbox'
const wrappedGenerator = new Throttle(sourceGenerator, speed)
```
### Synchronized
Fork a generator. The rules are:
- if the source returns, all the forks return
- if the forks error, all the forks error with the same error
- if a fork returns, it is stopped, but the generator continue with the other
- if a fork errors, it is stopped, but the generator continue with the other
- if all the forks return, the source is stopped
- if all the forks error, the source is errored with the last error
- the source start producing a packet when the fastest forked ask for it
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
```ts
import { Synchronized } from '@vates/generator-toolbox'
async function consume(generator: AsyncGenerator) {
for await (const val of generator) {
console.log({ val })
}
}
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await Promise.all([consume(first), consume(second)])
```
Note: you can stop early a generator by calling `generator.return()`, and you can stop in in error by calling `generator.throw(error)`
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[MIT](https://spdx.org/licenses/MIT) © [Vates SAS](https://vates.fr)

View File

@@ -1,6 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended)

View File

@@ -1,38 +0,0 @@
{
"name": "@vates/generator-toolbox",
"version": "1.1.0",
"main": "dist/index.mjs",
"license": "MIT",
"private": false,
"type": "module",
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^6.0.1",
"@types/node": "^20.6",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"prebuild": "rimraf dist/",
"predev": "yarn run prebuild",
"test": "tsc && node --test **/*.test.mjs",
"clean": "rimraf dist/"
},
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/generator-toolbox",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/generator-toolbox",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"engines": {
"node": ">=20.18"
}
}

View File

@@ -1,3 +0,0 @@
export { Synchronized } from './synchronized.mjs'
export { Throttle } from './throttle.mjs'
export { Timeout } from './timeout.mjs'

View File

@@ -1,150 +0,0 @@
import assert from 'node:assert'
export class Synchronized<T, TReturn, TNext> {
#source: AsyncGenerator<T, TReturn, TNext>
#forks = new Map<string, Forked<T, TReturn, TNext>>()
#removedForks = new Set<string>()
#waitingForks = new Set<string>()
#started = false
#nextValueForksReady?: {
promise: Promise<IteratorResult<T>>
forksWaitingReject: (error: Error) => void
forksWaitingResolve: () => void
}
constructor(source: AsyncGenerator<T, TReturn, TNext>) {
this.#source = source
}
fork(uid: string): AsyncGenerator<T, TReturn, TNext> {
assert.strictEqual(this.#started, false, `can't create a fork after consuming the data`)
assert.strictEqual(this.#forks.has(uid), false, `a fork with id ${uid} already exists`)
const fork = new Forked<T, TReturn, TNext>(this, uid)
this.#forks.set(uid, fork)
return fork
}
async #resolveWhenAllForksReady(): Promise<IteratorResult<T>> {
if (!this.#nextValueForksReady) {
throw new Error('Can t wait forks if there are noone waiting')
}
const { promise, forksWaitingResolve } = this.#nextValueForksReady
if (this.#waitingForks.size === this.#forks.size) {
// reset value
this.#waitingForks.clear()
this.#nextValueForksReady = undefined
forksWaitingResolve() // for the other forks waiting
}
return promise
}
async next(uid: string): Promise<IteratorResult<T>> {
if (this.#removedForks.has(uid)) {
return { done: true, value: undefined }
}
if (!this.#forks.has(uid)) {
throw new Error(`trying to advance fork ${uid} that is not a fork of this one`)
}
if (this.#waitingForks.has(uid)) {
throw new Error(`Fork ${uid} is already waiting`)
}
this.#started = true
if (this.#nextValueForksReady === undefined) {
let forksWaitingResolve = () => {}
let forksWaitingReject: (reason?: Error) => void = () => {}
const next = this.#source.next().catch(async error => {
const e = new Error(`Error in the source generator ${error.message}`, { cause: error })
forksWaitingReject(e)
// source has failed, kill everything, and stop the forks
for (const uid of [...this.#forks.keys()]) {
await this.remove(uid, error)
}
})
const promise = Promise.all([
next,
new Promise((_resolve, _reject) => {
forksWaitingResolve = () => _resolve(undefined)
forksWaitingReject = _reject
}),
]).then(([_]) => _ as IteratorResult<T>)
this.#nextValueForksReady = { promise, forksWaitingResolve, forksWaitingReject }
}
this.#waitingForks.add(uid)
return this.#resolveWhenAllForksReady()
}
async remove(uid: string, error?: Error): Promise<IteratorResult<T>> {
const fork = this.#forks.get(uid)
if (fork === undefined) {
if (this.#removedForks.has(uid)) {
// already removed
return { done: true, value: undefined }
}
throw new Error(`trying to remove fork with uid ${uid} that is not a fork of this one`)
}
this.#forks.delete(uid)
this.#waitingForks.delete(uid)
this.#removedForks.add(uid)
try {
if (error === undefined) {
await fork.return()
} else {
await fork.throw(error)
}
} catch (cleaningError) {
console.error('Error while cleaning the forked', {
cleaningError,
sourceError: error,
})
}
if (this.#forks.size === 0) {
if (error === undefined) {
await this.#source.return(undefined as TReturn)
} else {
await this.#source.throw(error)
}
// Reject any pending forks waiting for the next value
if (this.#nextValueForksReady) {
this.#nextValueForksReady.forksWaitingReject(new Error('Source generator terminated.', { cause: error }))
this.#nextValueForksReady = undefined
}
// clear state
this.#removedForks.clear()
this.#waitingForks.clear()
} else {
// this fork was maybe blocking the others
if (this.#nextValueForksReady) {
await this.#resolveWhenAllForksReady()
}
}
return { done: true, value: undefined }
}
}
class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
#parent: Synchronized<T, TReturn, TNext>
#uid: string
constructor(parent: Synchronized<T, TReturn, TNext>, uid: string) {
this.#parent = parent
this.#uid = uid
}
next(): Promise<IteratorResult<T>> {
return this.#parent.next(this.#uid)
}
async return(): Promise<IteratorResult<T>> {
return this.#parent.remove(this.#uid)
}
async throw(e: Error): Promise<IteratorResult<T>> {
return this.#parent.remove(this.#uid, e)
}
[Symbol.asyncIterator](): AsyncGenerator<T, TReturn, TNext> {
return this
}
}

View File

@@ -1,173 +0,0 @@
import assert from 'node:assert'
import { suite, test } from 'node:test'
import { Synchronized } from './synchronized.mjs'
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (_val: unknown) => {}) {
for (let i = 0; i < end; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
onYielded(i)
progress.yielded = i
}
return progress
}
async function consume(
iterable: AsyncGenerator,
delay = 2500,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
onConsumed = (_val: unknown, _iterable: AsyncGenerator) => Promise.resolve(false)
) {
for await (const val of iterable) {
await new Promise(resolve => setTimeout(resolve, delay))
if (await onConsumed(val, iterable)) {
iterable.return(undefined)
}
}
}
suite('success', () => {
test('for-await-of break stops the source generator', async () => {
let sourceClosed = false
const source = (async function* () {
try {
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 10))
yield i
}
} finally {
sourceClosed = true
}
})()
const forker = new Synchronized(source)
const fork = forker.fork('first')
// for-await-of must call fork.return() on break, which propagates to the source
for await (const val of fork) {
if (val === 2) break
}
assert.strictEqual(sourceClosed, true)
})
test('if works with multiple consumer', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(3, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await Promise.all([consume(first, 50), consume(second, 500)])
assert.strictEqual(progress.yielded, 2)
})
test('if works with one consumer returning', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(10, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await new Promise(resolve => setTimeout(resolve, 500))
await Promise.all([
consume(first, 1),
consume(second, 1, async (val: unknown) => {
return Promise.resolve(val === 2)
}),
])
assert.strictEqual(progress.yielded, 9)
})
test('if works with all consumers returning', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(10, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await new Promise(resolve => setTimeout(resolve, 500))
await Promise.all([
consume(first, 1, async (val: unknown) => {
return Promise.resolve(val === 5)
}),
consume(second, 1, async (val: unknown) => {
return Promise.resolve(val === 2)
}),
])
assert.strictEqual(progress.yielded, 4)
})
})
suite('error', () => {
test('it fails all if error on source', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(5, progress, (val: unknown) => {
if (val === 2) {
throw new Error('source stop')
}
})
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
await assert.rejects(Promise.all([consume(first, 1), consume(second, 1)]))
assert.strictEqual(progress.yielded, 1)
})
test('it flows until the end if one consumer throws', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(10, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
const firstConsume = consume(first, 1)
const secondConsume = assert.rejects(() =>
consume(second, 1, async (val: unknown, iterable) => {
if (val === 1) {
await iterable.throw(new Error('error on second'))
}
return false
})
)
await Promise.allSettled([firstConsume, secondConsume])
assert.strictEqual(progress.yielded, 9)
})
test('it throw if ALL consumer throws', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(5, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
const second = forker.fork('second')
const firstConsume = consume(first, 1, async (val: unknown, iterable) => {
if (val === 3) {
await iterable.throw(new Error('error on first'))
}
return false
})
const secondConsume = consume(second, 1, async (val: unknown, iterable) => {
if (val === 1) {
await iterable.throw(new Error('error on second'))
}
return false
})
await assert.rejects(Promise.all([firstConsume, secondConsume]))
assert.strictEqual(progress.yielded, 2)
})
test('It should not allow a new fork after the start', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(5, progress)
const forker = new Synchronized(generator)
const first = forker.fork('first')
await first.next()
assert.throws(() => forker.fork('second'))
})
test('It should not allow fork with the same uid', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(5, progress)
const forker = new Synchronized(generator)
forker.fork('first')
forker.fork('second')
assert.throws(() => forker.fork('first'))
})
})

View File

@@ -1,68 +0,0 @@
import assert from 'node:assert'
/**
*
* This class will throttle the production of a group of generators
*
* limits: depends on the event loop, so don't expect it to be very precise
* works better with more small blocks
* The source generator must yield object with a length property
* Changing the speed will only be takin into account fot the next packets asked
*/
export class Throttle {
#previousSlot = 0
#bytesPerSecond: number | (() => number)
get speed(): number {
let speed: number
if (typeof this.#bytesPerSecond === 'function') {
speed = this.#bytesPerSecond()
} else {
speed = this.#bytesPerSecond
}
assert.ok(speed >= 0, `speed must be positive, ${speed} computed`)
return speed
}
constructor(speed: number | (() => number)) {
this.#bytesPerSecond = speed
}
getNextSlot(length: number): { timeout?: ReturnType<typeof setTimeout>; promise?: Promise<unknown> } {
assert.notStrictEqual(length, undefined, `throttled stream need to expose a length property }`)
assert.ok(length > 0, `throttled stream must expose a positive length property , ${length} given }`)
if (this.speed === 0) {
return {}
}
const previous = this.#previousSlot
const nextSlot = Math.round(previous + (length * 1000) / this.speed)
if (nextSlot < Date.now()) {
// we're above the limit, go now
this.#previousSlot = Date.now()
return {}
}
this.#previousSlot = nextSlot
// wait till the next slot
// it won't be extremely precise since the event loop is not
let timeout: ReturnType<typeof setTimeout> | undefined = undefined
const promise = new Promise(function (resolve) {
timeout = setTimeout(resolve, nextSlot - Date.now())
})
return { promise, timeout }
}
async *createThrottledGenerator(source: AsyncGenerator<{ length: number }>): AsyncGenerator<{ length: number }> {
let timeout: ReturnType<typeof setTimeout> | undefined
try {
for await (const value of source) {
const res = this.getNextSlot(value.length)
timeout = res.timeout
// wait for the time slot before yielding the data
if (res.promise !== undefined) {
await res.promise
}
yield value
}
} finally {
clearTimeout(timeout)
}
}
}

View File

@@ -1,63 +0,0 @@
import { Throttle } from './throttle.mjs'
import { suite, test } from 'node:test'
import assert from 'node:assert'
async function* makeGenerator(chunkSize = 1024 * 1024, nbChunks = 100) {
for (let i = 0; i < nbChunks; i++) {
yield Buffer.allocUnsafe(chunkSize)
}
}
async function consumes(generator: AsyncGenerator) {
// eslint-disable-next-line no-empty,@typescript-eslint/no-unused-vars
for await (const data of generator) {
}
}
suite('it throttle one', () => {
test('base test', async () => {
// 10MB/s
const throttler = new Throttle(10 * 1024 * 1024)
const throttled = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 20))
const start = Date.now()
await consumes(throttled)
const end = Date.now()
assert.strictEqual(Math.round((end - start) / 1000), 2)
})
test('variable speed test', async () => {
// 10MB/s
const throttler = new Throttle(() => 10 * 1024 * 1024)
const throttled = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 20))
const start = Date.now()
await consumes(throttled)
const end = Date.now()
assert.strictEqual(Math.round((end - start) / 1000), 2)
})
test('multiple generator test', async () => {
// 10MB/s
const throttler = new Throttle(10 * 1024 * 1024)
const first = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
const second = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
const start = Date.now()
await Promise.all([consumes(first), consumes(second)])
const end = Date.now()
assert.strictEqual(Math.round((end - start) / 1000), 2)
})
test(`it doesn't accept negative value`, async () => {
// 10MB/s
let throttler = new Throttle(-10 * 1024 * 1024)
let generator = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
await assert.rejects(() => consumes(generator))
let index = 5
throttler = new Throttle(() => {
index--
return index > 0 ? 10 * 1024 * 1024 : -10 * 1024 * 1024
})
generator = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
await assert.rejects(() => consumes(generator))
})
})

View File

@@ -1,49 +0,0 @@
import assert from 'node:assert'
const TIMEOUT = Symbol('timeout')
export class Timeout<T> implements AsyncGenerator {
#source: AsyncGenerator<T>
#timeout: number
constructor(source: AsyncGenerator<T>, timeout: number) {
assert.ok(timeout > 0, `Timeout value must be positive, ${timeout} received`)
this.#source = source
this.#timeout = timeout
}
async next(): Promise<IteratorResult<T>> {
let timeoutHandle: ReturnType<typeof setTimeout>
const sourceNext = this.#source.next()
const result = await Promise.race([
sourceNext.then(res => {
clearTimeout(timeoutHandle)
return res
}),
new Promise<typeof TIMEOUT>(resolve => {
timeoutHandle = setTimeout(() => resolve(TIMEOUT), this.#timeout)
}),
])
if (result === TIMEOUT) {
// stop the source once the in-flight next settles to avoid data loss on the next call
// we don't want to await for the end of timeouted code
sourceNext.then(
() => this.#source.return(undefined).catch(() => {}),
() => {} // source already errored, nothing to clean up
)
throw new Error('Timeout reached')
}
return result
}
return(): Promise<IteratorResult<T>> {
return this.#source.return(undefined)
}
throw(e: Error): Promise<IteratorResult<T>> {
return this.#source.throw(e)
}
[Symbol.asyncIterator](): AsyncGenerator<T> {
return this
}
}

View File

@@ -1,87 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert'
import { Timeout } from './timeout.mjs'
describe('Timeout class', () => {
it('should reject a timeout with a negative or zero value ', async () => {
// Create a mock async generator that never resolves
const mockGenerator = (async function* () {
yield 1
})()
assert.throws(() => new Timeout(mockGenerator, 0))
assert.throws(() => new Timeout(mockGenerator, -10))
})
it('should resolve with the value from the source generator if it completes before the timeout', async () => {
// Create a mock async generator that resolves immediately
const mockGenerator = (async function* () {
yield 1
})()
const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
const result = await timeout.next()
assert.strictEqual(result.value, 1)
assert.strictEqual(result.done, false)
})
it('should reject with a timeout error if the source generator takes too long', async () => {
// Create a mock async generator that never resolves
// eslint-disable-next-line require-yield
const mockGenerator = (async function* () {
await new Promise(() => {}) // Never resolves
})()
const timeout = new Timeout(mockGenerator, 10) // 10ms timeout
await assert.rejects(timeout.next(), 'Expected timeout error was not thrown')
})
it('should allow iteration using for-await-of', async () => {
// Create a mock async generator that yields values
const mockGenerator: AsyncGenerator<number> = (async function* () {
yield 1
yield 2
yield 3
})()
const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
const results: Array<number> = []
for await (const value of timeout) {
results.push(value as number)
}
assert.deepStrictEqual(results, [1, 2, 3])
})
it('should handle the throw method correctly', async () => {
// Create a mock async generator with a throw method
const mockGenerator = (async function* () {
yield 1
})()
const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
await assert.rejects(timeout.throw(new Error('Test error')), 'Expected error was not thrown')
})
it('should close the source generator after a timeout', async () => {
let sourceClosed = false
const slowSource = (async function* () {
try {
// resolves after 100ms so the in-flight next() eventually settles
await new Promise(resolve => setTimeout(resolve, 100))
yield 1
} finally {
sourceClosed = true
}
})()
const timeout = new Timeout(slowSource, 10) // fires before source yields
await assert.rejects(() => timeout.next(), /Timeout reached/)
// wait for the in-flight next() to settle so source.return() can be called
await new Promise(resolve => setTimeout(resolve, 200))
assert.strictEqual(sourceClosed, true)
})
})

View File

@@ -1,13 +0,0 @@
{
"compilerOptions": {
"target": "es2023",
"module": "ESNext",
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
"moduleResolution": "bundler",
"strict": true,
"declaration": true, // Generates `.d.ts` files for type safety
"outDir": "dist" // Outputs compiled code to the `dist` directory
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

View File

@@ -1,4 +1,4 @@
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.

View File

@@ -16,7 +16,7 @@ npm install --save @vates/json-hash
## Usage
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.

View File

@@ -1,132 +0,0 @@
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
import NbdClient from '@vates/nbd-client'
/**
* @typedef {{ offset: number, length: number }} DataRange
*/
/**
* @extends {RandomAccessDisk}
*/
export class NbdDisk extends RandomAccessDisk {
#nbdInfos
/** @type {NbdClient|undefined} */
#nbdClient
/** @type {Array<DataRange> | undefined} */
#dataMap
/** @type {number} */
#blockSize
constructor(nbdInfos, blockSize, { dataMap } = {}) {
super()
this.#blockSize = blockSize
this.#nbdInfos = nbdInfos
this.#dataMap = dataMap && this.#processDatamap(dataMap)
}
#processDatamap(rawDataMap) {
return rawDataMap
.filter(({ type }) => type === 0)
.map(({ offset, length }) => ({ offset, length }))
.sort(({ offset: offset1 }, { offset: offset2 }) => offset1 - offset2)
}
/**
* @param {number} index
* @returns {Promise<DiskBlock>}
*/
async readBlock(index) {
if (!this.#nbdClient) {
throw new Error("can't readBlock before init")
}
let data = await this.#nbdClient.readBlock(index, this.getBlockSize())
if (data.length !== this.getBlockSize()) {
// this condition only work on non aligned disks
// we won't accept truncated block from unaligned disks
if (index === Math.ceil(this.getVirtualSize() / this.getBlockSize()) - 1) {
// last block add zeros at the end
data = Buffer.concat([data], this.getBlockSize())
} else {
throw new Error(`Block ${index} is not at the right size, expecting ${this.getBlockSize()}, got ${data.length}`)
}
}
return {
index,
data,
}
}
/**
* @returns {number}
*/
getVirtualSize() {
if (!this.#nbdClient) {
throw new Error("can't get size before init")
}
return Number(this.#nbdClient.exportSize)
}
/**
* @returns {number}
*/
getBlockSize() {
return this.#blockSize
}
/**
* @returns {Promise<void>}
*/
async init() {
this.#nbdClient = new NbdClient(this.#nbdInfos)
await this.#nbdClient.connect()
if (this.#dataMap === undefined) {
this.#dataMap = this.#processDatamap(await this.#nbdClient.getMap())
}
}
/**
* @returns {Promise<void>}
*/
async close() {
await this.#nbdClient?.disconnect()
}
/**
* @returns {boolean}
*/
isDifferencing() {
return false
}
/**
* @returns {Array<number>}
*/
getBlockIndexes() {
const indexes = new Set()
this.#dataMap?.forEach(({ offset, length }) => {
if (length === 0) return
const firstBlockIndex = Math.floor(offset / this.getBlockSize())
const lastBlockIndex = Math.floor((offset + length - 1) / this.getBlockSize())
for (let blockIndex = firstBlockIndex; blockIndex <= lastBlockIndex; blockIndex++) {
indexes.add(blockIndex)
}
})
return [...indexes].sort((a, b) => a - b)
}
/**
* @param {number} index
* @returns {boolean}
*/
hasBlock(index) {
if (!this.#dataMap) {
throw new Error("can't hasBlock before init")
}
const blockStart = index * this.getBlockSize()
const blockEnd = (index + 1) * this.getBlockSize()
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && offset < blockEnd)
}
}

View File

@@ -1,6 +1,6 @@
export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negotiation
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
export const NBD_OPT_EXPORT_NAME = 1
export const NBD_OPT_ABORT = 2
export const NBD_OPT_LIST = 3

View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'
import { Socket } from 'node:net'
import { connect } from 'node:tls'
import { pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
import { readChunkStrict } from '@vates/read-chunk'
import { createLogger } from '@xen-orchestra/log'
import {
@@ -20,8 +20,6 @@ import {
OPTS_MAGIC,
NBD_CMD_DISC,
} from './constants.mjs'
import { spawn } from 'node:child_process'
const { warn } = createLogger('vates:nbd-client')
// documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -44,9 +42,9 @@ export default class NbdClient {
// AFAIK, there is no guaranty the server answers in the same order as the queries
// so we handle a backlog of command waiting for response and handle concurrency manually
#waitingForResponse // there is already a listener waiting for a response
#waitingForResponse // there is already a listenner waiting for a response
#nextCommandQueryId = BigInt(0)
#commandQueryBacklog // map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
#commandQueryBacklog // map of command waiting for an response queryId => { size/*in byte*/, resolve, reject}
#connected = false
#reconnectingPromise
@@ -197,7 +195,7 @@ export default class NbdClient {
}
// send export name we want to access.
// it's implicitly closing the negotiation phase.
// it's implictly closing the negociation phase.
await this.#write(OPTS_MAGIC)
await this.#writeInt32(NBD_OPT_EXPORT_NAME)
const exportNameBuffer = Buffer.from(this.#exportName)
@@ -220,22 +218,8 @@ export default class NbdClient {
}
#write(buffer) {
let timeout
const messageTimeout = this.#messageTimeout
const socket = this.#serverSocket
return Promise.race([
new Promise((resolve, reject) => {
timeout = setTimeout(() => {
reject(new Error('timeout'))
}, messageTimeout)
}),
new Promise(resolve =>
socket.write(buffer, () => {
clearTimeout(timeout)
resolve()
})
),
])
const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
return pTimeout.call(promise, this.#messageTimeout)
}
async #readInt32() {
@@ -359,40 +343,4 @@ export default class NbdClient {
},
})
}
/**
* returns the map of the file with holes, zeros and data, useful to handle efficiently sparse source
* to implement this internally: use structure response if the server supports it, and then ask for BLOCK_STATUS *
*
* @returns {Promise<{ offset: number, length: number, type: number }[]>}
* A promise that resolves to an array where each object represents a segment:
* - `offset` — The byte offset from the start.
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type ( 0 means data).
*/
/* async */ getMap() {
return new Promise((resolve, reject) => {
const process = spawn('nbdinfo', [
'--json',
'--map',
`nbd://${this.#serverAddress}:${this.#serverPort}/${encodeURIComponent(this.#exportName)}`,
])
let text = ''
let errText = ''
process.stdout.on('data', data => (text += data))
process.stderr.on('data', data => (errText += data))
process.on('error', reject)
process.on('close', code => {
if (code !== 0) {
return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
}
try {
const json = JSON.parse(text)
resolve(json)
} catch (error) {
reject(new Error(`${error} ${text}`))
}
})
})
}
}

View File

@@ -30,8 +30,6 @@ export default class MultiNbdClient {
* open nbdConcurrency connections to NBD servers
* it must obtain at least one connection to succeed
* it tries to spread connections on multiple host
*
* @returns {Promise<void>}
*/
async connect() {
const candidates = [...this.#settings]
@@ -70,9 +68,7 @@ export default class MultiNbdClient {
await _connect()
}
if (this.#clients.length === 0) {
const error = new Error(`Fail to connect to any Nbd client`, { nbdInfos: this.#settings })
error.code = 'NO_NBD_AVAILABLE'
throw error
throw new Error(`Fail to connect to any Nbd client`)
}
if (this.#clients.length < this.#nbdConcurrency) {
warn(
@@ -81,30 +77,17 @@ export default class MultiNbdClient {
}
}
/**
* @returns {Promise<void>}
*/
async disconnect() {
await asyncEach(this.#clients, client => client.disconnect(), {
stopOnError: false,
})
}
/**
*
* @param {number} index
* @param {number} size
* @returns {Promise<Buffer>}
*/
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
const clientId = index % this.#clients.length
return this.#clients[clientId].readBlock(index, size)
}
/**
*
* @param {AsyncGenerator<Buffer>} indexGenerator
*/
async *readBlocks(indexGenerator) {
// default : read all blocks
const readAhead = []
@@ -129,17 +112,4 @@ export default class MultiNbdClient {
yield readAhead.shift()
}
}
/**
* returns the map of the file with holes, zeros and data, useful to handle efficiently sparse source *
*
* @returns {Promise<{ offset: number, length: number, type: number }[]>}
* A promise that resolves to an array where each object represents a segment:
* - `offset` — The byte offset from the start.
* - `length` — The size of the segment in bytes.
* - `type` — A numeric code indicating the segment type (0 means no data).
*/
async getMap() {
// ask the map from one of the connected client
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap()
}
}

View File

@@ -13,19 +13,18 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.2.3",
"version": "3.1.2",
"engines": {
"node": ">=14.0"
},
"main": "./index.mjs",
"dependencies": {
"@vates/async-each": "^1.0.1",
"@vates/read-chunk": "^1.2.1",
"@vates/async-each": "^1.0.0",
"@vates/read-chunk": "^1.2.0",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/disk-transform": "^1.2.1",
"@xen-orchestra/log": "^0.7.1",
"promise-toolbox": "^0.21.0",
"xen-api": "^4.7.6"
"xen-api": "^4.7.0"
},
"devDependencies": {
"tap": "^18.7.0",

View File

@@ -24,7 +24,7 @@ $ npm install node-vsphere-soap --save
## Sample Code
### To connect to a vCenter server
### To connect to a vCenter server:
var nvs = require('node-vsphere-soap');
var vc = new nvs.Client(host, user, password, sslVerify);
@@ -54,7 +54,7 @@ $ npm install node-vsphere-soap --save
- userName - username of authenticated user
- fullName - full name of authenticated user
### To run a command
### To run a command:
var vcCmd = vc.runCommand( commandToRun, arguments );
vcCmd.once('result', function( result, raw, soapHeader) {

View File

@@ -7,7 +7,7 @@
This file creates the Client class
- when the class is instantiated, a connection will be made to the ESXi/vCenter server to verify that the creds are good
- upon a bad login, the connection will be terminated
- upon a bad login, the connnection will be terminated
*/

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/node-vsphere-soap",
"version": "2.1.2",
"version": "2.1.1",
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
"main": "lib/client.mjs",
"author": "reedog117",
@@ -12,7 +12,7 @@
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"axios": "^1.8.0",
"axios": "^1.4.0",
"soap": "^1.0.0",
"soap-cookie": "^0.10.1"
},

View File

@@ -1,6 +1,6 @@
### Usual workflow
> This section presents how this library should be used to implement a classic two factor authentication.
> This section presents how this library should be used to implement a classic two factor authentification.
#### Setup

View File

@@ -18,7 +18,7 @@ npm install --save @vates/otp
### Usual workflow
> This section presents how this library should be used to implement a classic two factor authentication.
> This section presents how this library should be used to implement a classic two factor authentification.
#### Setup

View File

@@ -71,17 +71,13 @@ exports.readChunk = readChunk
* If `size` bytes are not available to be read, the returned promise is rejected.
*
* @param {Readable} stream - A readable stream to read from.
* @param {number?} [size] - The number of bytes to read for binary streams (ignored for object streams).
* @param {number} [size] - The number of bytes to read for binary streams (ignored for object streams).
* @returns {Promise<Buffer|string|unknown>} - A Promise that resolves to the read chunk.
*/
exports.readChunkStrict = async function readChunkStrict(stream, size) {
const chunk = await readChunk(stream, size)
if (chunk === null) {
if (size === undefined) {
throw new Error(`stream has ended without data`)
} else {
throw new Error(`stream has ended without data, was looking for ${size} bytes`)
}
throw new Error('stream has ended without data')
}
if (size !== undefined && chunk.length !== size) {

View File

@@ -19,7 +19,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "1.2.1",
"version": "1.2.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -63,16 +63,16 @@ task.failure(error)
Inside a task:
```js
// the abort signal of the current task if any; otherwise, is `undefined`
// the abort signal of the current task if any, otherwise is `undefined`
Task.abortSignal
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.info(message, data)
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.warning(message, data)
// attaches a property to the current task if any; otherwise, does nothing
// attaches a property to the current task if any, otherwise does nothing
//
// the latest value takes precedence
//
@@ -149,7 +149,7 @@ const onProgress = makeOnProgress({
// This function is called each time a root task or a subtask is updated.
//
// `taskLog.$root` can be used to unconditionally access the root task.
// `taskLog.$root` can be used to uncondionally access the root task.
onTaskUpdate(taskLog) {},
})

View File

@@ -79,16 +79,16 @@ task.failure(error)
Inside a task:
```js
// the abort signal of the current task if any; otherwise, is `undefined`
// the abort signal of the current task if any, otherwise is `undefined`
Task.abortSignal
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.info(message, data)
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.warning(message, data)
// attaches a property to the current task if any; otherwise, does nothing
// attaches a property to the current task if any, otherwise does nothing
//
// the latest value takes precedence
//
@@ -165,7 +165,7 @@ const onProgress = makeOnProgress({
// This function is called each time a root task or a subtask is updated.
//
// `taskLog.$root` can be used to unconditionally access the root task.
// `taskLog.$root` can be used to uncondionally access the root task.
onTaskUpdate(taskLog) {},
})

View File

@@ -29,7 +29,7 @@ class Task {
static get abortSignal() {
const task = getTask()
if (task !== undefined) {
return task._abortController.signal
return task.#abortController.signal
}
}
@@ -65,11 +65,8 @@ class Task {
}
}
// These two properties should not be used outside the class
//
// To ensure compatibility between multiple versions of this lib, they are kept public for now
_abortController = new AbortController()
_onProgress
#abortController = new AbortController()
#onProgress
get id() {
return (this.id = Math.random().toString(36).slice(2))
@@ -89,23 +86,23 @@ class Task {
this.#startData = { properties }
if (onProgress !== undefined) {
this._onProgress = onProgress
this.#onProgress = onProgress
} else {
const parent = getTask()
if (parent !== undefined) {
const { signal } = parent._abortController
const { signal } = parent.#abortController
signal.addEventListener('abort', () => {
this._abortController.abort(signal.reason)
this.#abortController.abort(signal.reason)
})
this._onProgress = parent._onProgress
this.#onProgress = parent.#onProgress
this.#startData.parentId = parent.id
} else {
this._onProgress = noop
this.#onProgress = noop
}
}
const { signal } = this._abortController
const { signal } = this.#abortController
signal.addEventListener('abort', () => {
if (this.status === PENDING) {
this.#maybeStart()
@@ -120,7 +117,7 @@ class Task {
}
abort(reason) {
this._abortController.abort(reason)
this.#abortController.abort(reason)
}
#emit(type, data) {
@@ -129,7 +126,7 @@ class Task {
data.id = this.id
data.timestamp = Date.now()
data.type = type
this._onProgress(data)
this.#onProgress(data)
}
#end(status, result) {
@@ -137,7 +134,7 @@ class Task {
assert.equal(this.#running, false)
this.#emit('end', { status, result })
this._onProgress = alreadyEnded
this.#onProgress = alreadyEnded
this.#status = status
}

View File

@@ -29,7 +29,7 @@ function createTask(opts) {
}
describe('Task', function () {
describe('constructor', function () {
describe('contructor', function () {
it('data properties are passed to the start event', async function () {
const properties = { foo: 0, bar: 1 }
const task = createTask({ properties })
@@ -265,7 +265,7 @@ describe('Task', function () {
assert.throws(() => task.set(name, value), { message: /^task has not started yet\b/ })
})
it(`emits a property message`, async function () {
it(`emits an property message`, async function () {
const task = createTask()
await task.run(async () => {
await Task.run(() => {
@@ -339,7 +339,7 @@ describe('Task', function () {
Task.warning('foo')
})
it('emits a warning message when run inside a task', async function () {
it('emits an warning message when run inside a task', async function () {
const task = createTask()
await task.run(() => {
Task.warning('foo')

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.6.2",
"version": "0.6.0",
"engines": {
"node": ">=14"
},

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,21 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
},
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,57 +0,0 @@
{
"name": "@vates/types",
"private": false,
"version": "1.21.0",
"main": "./dist/index.mjs",
"exports": {
".": {
"default": "./dist/index.mjs",
"types": "./dist/index.d.mts"
},
"./xo": "./dist/xo.d.mts",
"./xo-app": "./dist/xo-app.d.mts",
"./xen-api": "./dist/xen-api.d.mts",
"./common": {
"default": "./dist/common.mjs",
"types": "./dist/common.d.mts"
},
"./lib/vates/*": {
"default": "./dist/lib/vates-*.mjs",
"type": "./dist/lib/vates-*.d.mts"
},
"./lib/xen-orchestra/*": {
"default": "./dist/lib/xen-orchestra-*.mjs",
"type": "./dist/lib/xen-orchestra-*.d.mts"
},
"./lib/complex-matcher": {
"default": "./dist/lib/complex-matcher.mjs",
"type": "./dist/lib/complex-matcher.d.mts"
}
},
"type": "module",
"devDependencies": {
"@eslint/js": "^9.19.0",
"typescript": "~5.6.3",
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",
"prepublishOnly": "yarn run build",
"postversion": "npm publish --access public"
},
"license": "AGPL-3.0-or-later",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/types",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/types",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"engines": {
"node": ">=18.18"
}
}

View File

@@ -1,742 +0,0 @@
import type { XoHost } from './xo.mjs'
declare const __brand: unique symbol
export type Branded<TBrand extends string, TType = string> = TType & { [__brand]: TBrand }
export const HOST_POWER_STATE = {
RUNNING: 'Running',
HALTED: 'Halted',
UNKNOWN: 'Unknown',
} as const
export type HOST_POWER_STATE = (typeof HOST_POWER_STATE)[keyof typeof HOST_POWER_STATE]
export const TASK_ALLOWED_OPERATIONS = {
CANCEL: 'cancel',
DESTROY: 'destroy',
} as const
export type TASK_ALLOWED_OPERATIONS = (typeof TASK_ALLOWED_OPERATIONS)[keyof typeof TASK_ALLOWED_OPERATIONS]
export const TASK_STATUS_TYPE = {
CANCELLED: 'cancelled',
CANCELLING: 'cancelling',
FAILURE: 'failure',
PENDING: 'pending',
SUCCESS: 'success',
} as const
export type TASK_STATUS_TYPE = (typeof TASK_STATUS_TYPE)[keyof typeof TASK_STATUS_TYPE]
export const EVENT_OPERATION = {
ADD: 'add',
DEL: 'del',
MOD: 'mod',
} as const
export type EVENT_OPERATION = (typeof EVENT_OPERATION)[keyof typeof EVENT_OPERATION]
export const POOL_ALLOWED_OPERATIONS = {
APPLY_UPDATES: 'apply_updates',
CERT_REFRESH: 'cert_refresh',
CLUSTER_CREATE: 'cluster_create',
CONFIGURE_REPOSITORIES: 'configure_repositories',
COPY_PRIMARY_HOST_CERTS: 'copy_primary_host_certs',
DESIGNATE_NEW_MASTER: 'designate_new_master',
EJECT: 'eject',
EXCHANGE_CA_CERTIFICATES_ON_JOIN: 'exchange_ca_certificates_on_join',
EXCHANGE_CERTIFICATES_ON_JOIN: 'exchange_certificates_on_join',
GET_UPDATES: 'get_updates',
HA_DISABLE: 'ha_disable',
HA_ENABLE: 'ha_enable',
SYNC_UPDATES: 'sync_updates',
TLS_VERIFICATION_ENABLE: 'tls_verification_enable',
} as const
export type POOL_ALLOWED_OPERATIONS = (typeof POOL_ALLOWED_OPERATIONS)[keyof typeof POOL_ALLOWED_OPERATIONS]
export const TELEMETRY_FREQUENCY = {
DAILY: 'daily',
MONTHLY: 'monthly',
WEEKLY: 'weekly',
} as const
export type TELEMETRY_FREQUENCY = (typeof TELEMETRY_FREQUENCY)[keyof typeof TELEMETRY_FREQUENCY]
export const UPDATE_SYNC_FREQUENCY = {
DAILY: 'daily',
WEEKLY: 'weekly',
} as const
export type UPDATE_SYNC_FREQUENCY = (typeof UPDATE_SYNC_FREQUENCY)[keyof typeof UPDATE_SYNC_FREQUENCY]
export const AFTER_APPLY_GUIDANCE = {
RESTART_HOST: 'restartHost',
RESTART_HVM: 'restartHVM',
RESTART_PV: 'restartPV',
RESTART_XAPI: 'restartXAPI',
} as const
export type AFTER_APPLY_GUIDANCE = (typeof AFTER_APPLY_GUIDANCE)[keyof typeof AFTER_APPLY_GUIDANCE]
export const UPDATE_AFTER_APPLY_GUIDANCE = {
RESTART_HOST: 'restartHost',
RESTART_HVM: 'restartHVM',
RESTART_PV: 'restartPV',
RESTART_XAPI: 'restartXAPI',
} as const
export type UPDATE_AFTER_APPLY_GUIDANCE = (typeof UPDATE_AFTER_APPLY_GUIDANCE)[keyof typeof UPDATE_AFTER_APPLY_GUIDANCE]
export const LIVEPATCH_STATUS = {
OK: 'ok',
OK_LIVEPATCH_COMPLETE: 'ok_livepatch_complete',
OK_LIVEPATCH_INCOMPLETE: 'ok_livepatch_incomplete',
} as const
export type LIVEPATCH_STATUS = (typeof LIVEPATCH_STATUS)[keyof typeof LIVEPATCH_STATUS]
export const VM_POWER_STATE = {
HALTED: 'Halted',
PAUSED: 'Paused',
RUNNING: 'Running',
SUSPENDED: 'Suspended',
} as const
export type VM_POWER_STATE = (typeof VM_POWER_STATE)[keyof typeof VM_POWER_STATE]
export const UPDATE_GUIDANCES = {
REBOOT_HOST: 'reboot_host',
REBOOT_HOST_ON_KERNEL_LIVEPATCH_FAILURE: 'reboot_host_on_kernel_livepatch_failure',
REBOOT_HOST_ON_LIVEPATCH_FAILURE: 'reboot_host_on_livepatch_failure',
REBOOT_HOST_ON_XEN_LIVEPATCH_FAILURE: 'reboot_host_on_xen_livepatch_failure',
RESTART_DEVICE_MODEL: 'restart_device_model',
RESTART_TOOLSTACK: 'restart_toolstack',
RESTART_VM: 'restart_vm',
} as const
export type UPDATE_GUIDANCES = (typeof UPDATE_GUIDANCES)[keyof typeof UPDATE_GUIDANCES]
export const ON_SOFTREBOOT_BEHAVIOR = {
DESTROY: 'destroy',
PRESERVE: 'preserve',
RESTART: 'restart',
SOFT_REBOOT: 'soft_reboot',
} as const
export type ON_SOFTREBOOT_BEHAVIOR = (typeof ON_SOFTREBOOT_BEHAVIOR)[keyof typeof ON_SOFTREBOOT_BEHAVIOR]
export const ON_NORMAL_EXIT = {
DESTROY: 'destroy',
RESTART: 'restart',
} as const
export type ON_NORMAL_EXIT = (typeof ON_NORMAL_EXIT)[keyof typeof ON_NORMAL_EXIT]
export const VM_OPERATIONS = {
ASSERT_OPERATION_VALID: 'assert_operation_valid',
AWAITING_MEMORY_LIVE: 'awaiting_memory_live',
CALL_PLUGIN: 'call_plugin',
CHANGING_DYNAMIC_RANGE: 'changing_dynamic_range',
CHANGING_MEMORY_LIMITS: 'changing_memory_limits',
CHANGING_MEMORY_LIVE: 'changing_memory_live',
CHANGING_NVRAM: 'changing_NVRAM',
CHANGING_SHADOW_MEMORY: 'changing_shadow_memory',
CHANGING_SHADOW_MEMORY_LIVE: 'changing_shadow_memory_live',
CHANGING_STATIC_RANGE: 'changing_static_range',
CHANGING_VCPUS: 'changing_VCPUs',
CHANGING_VCPUS_LIVE: 'changing_VCPUs_live',
CHECKPOINT: 'checkpoint',
CLEAN_REBOOT: 'clean_reboot',
CLEAN_SHUTDOWN: 'clean_shutdown',
CLONE: 'clone',
COPY: 'copy',
CREATE_TEMPLATE: 'create_template',
CREATE_VTPM: 'create_vtpm',
CSVM: 'csvm',
DATA_SOURCE_OP: 'data_source_op',
DESTROY: 'destroy',
EXPORT: 'export',
GET_BOOT_RECORD: 'get_boot_record',
HARD_REBOOT: 'hard_reboot',
HARD_SHUTDOWN: 'hard_shutdown',
IMPORT: 'import',
MAKE_INTO_TEMPLATE: 'make_into_template',
METADATA_EXPORT: 'metadata_export',
MIGRATE_SEND: 'migrate_send',
PAUSE: 'pause',
POOL_MIGRATE: 'pool_migrate',
POWER_STATE_RESET: 'power_state_reset',
PROVISION: 'provision',
QUERY_SERVICES: 'query_services',
RESUME: 'resume',
RESUME_ON: 'resume_on',
REVERT: 'revert',
REVERTING: 'reverting',
SEND_SYSRQ: 'send_sysrq',
SEND_TRIGGER: 'send_trigger',
SHUTDOWN: 'shutdown',
SNAPSHOT: 'snapshot',
SNAPSHOT_WITH_QUIESCE: 'snapshot_with_quiesce',
START: 'start',
START_ON: 'start_on',
SUSPEND: 'suspend',
UNPAUSE: 'unpause',
UPDATE_ALLOWED_OPERATIONS: 'update_allowed_operations',
} as const
export type VM_OPERATIONS = (typeof VM_OPERATIONS)[keyof typeof VM_OPERATIONS]
export const ON_CRASH_BEHAVIOUR = {
COREDUMP_AND_DESTROY: 'coredump_and_destroy',
COREDUMP_AND_RESTART: 'coredump_and_restart',
DESTROY: 'destroy',
PRESERVE: 'preserve',
RENAME_RESTART: 'rename_restart',
RESTART: 'restart',
} as const
export type ON_CRASH_BEHAVIOUR = (typeof ON_CRASH_BEHAVIOUR)[keyof typeof ON_CRASH_BEHAVIOUR]
export const DOMAIN_TYPE = {
HVM: 'hvm',
PV: 'pv',
PVH: 'pvh',
PV_IN_PVH: 'pv_in_pvh',
UNSPECIFIED: 'unspecified',
} as const
export type DOMAIN_TYPE = (typeof DOMAIN_TYPE)[keyof typeof DOMAIN_TYPE]
export const TRISTATE_TYPE = {
NO: 'no',
UNSPECIFIED: 'unspecified',
YES: 'yes',
} as const
export type TRISTATE_TYPE = (typeof TRISTATE_TYPE)[keyof typeof TRISTATE_TYPE]
export const VMPP_BACKUP_TYPE = {
CHECKPOINT: 'checkpoint',
SNAPSHOT: 'snapshot',
} as const
export type VMPP_BACKUP_TYPE = (typeof VMPP_BACKUP_TYPE)[keyof typeof VMPP_BACKUP_TYPE]
export const VMPP_BACKUP_FREQUENCY = {
DAILY: 'daily',
HOURLY: 'hourly',
WEEKLY: 'weekly',
} as const
export type VMPP_BACKUP_FREQUENCY = (typeof VMPP_BACKUP_FREQUENCY)[keyof typeof VMPP_BACKUP_FREQUENCY]
export const VMPP_ARCHIVE_FREQUENCY = {
ALWAYS_AFTER_BACKUP: 'always_after_backup',
DAILY: 'daily',
NEVER: 'never',
WEEKLY: 'weekly',
} as const
export type VMPP_ARCHIVE_FREQUENCY = (typeof VMPP_ARCHIVE_FREQUENCY)[keyof typeof VMPP_ARCHIVE_FREQUENCY]
export const VMPP_ARCHIVE_TARGET_TYPE = {
CIFS: 'cifs',
NFS: 'nfs',
NONE: 'none',
} as const
export type VMPP_ARCHIVE_TARGET_TYPE = (typeof VMPP_ARCHIVE_TARGET_TYPE)[keyof typeof VMPP_ARCHIVE_TARGET_TYPE]
export const VMSS_FREQUENCY = {
DAILY: 'daily',
HOURLY: 'hourly',
WEEKLY: 'weekly',
} as const
export type VMSS_FREQUENCY = (typeof VMSS_FREQUENCY)[keyof typeof VMSS_FREQUENCY]
export const VMSS_TYPE = {
CHECKPOINT: 'checkpoint',
SNAPSHOT: 'snapshot',
SNAPSHOT_WITH_QUIESCE: 'snapshot_with_quiesce',
} as const
export type VMSS_TYPE = (typeof VMSS_TYPE)[keyof typeof VMSS_TYPE]
export const VM_APPLIANCE_OPERATION = {
CLEAN_SHUTDOWN: 'clean_shutdown',
HARD_SHUTDOWN: 'hard_shutdown',
SHUTDOWN: 'shutdown',
START: 'start',
} as const
export type VM_APPLIANCE_OPERATION = (typeof VM_APPLIANCE_OPERATION)[keyof typeof VM_APPLIANCE_OPERATION]
export const HOST_ALLOWED_OPERATIONS = {
APPLY_UPDATES: 'apply_updates',
ENABLE: 'enable',
EVACUATE: 'evacuate',
POWER_ON: 'power_on',
PROVISION: 'provision',
REBOOT: 'reboot',
SHUTDOWN: 'shutdown',
VM_MIGRATE: 'vm_migrate',
VM_RESUME: 'vm_resume',
VM_START: 'vm_start',
} as const
export type HOST_ALLOWED_OPERATIONS = (typeof HOST_ALLOWED_OPERATIONS)[keyof typeof HOST_ALLOWED_OPERATIONS]
export const LATEST_SYNCED_UPDATES_APPLIED_STATE = {
NO: 'no',
UNKNOWN: 'unknown',
YES: 'yes',
} as const
export type LATEST_SYNCED_UPDATES_APPLIED_STATE =
(typeof LATEST_SYNCED_UPDATES_APPLIED_STATE)[keyof typeof LATEST_SYNCED_UPDATES_APPLIED_STATE]
export const HOST_DISPLAY = {
DISABLED: 'disabled',
DISABLE_ON_REBOOT: 'disable_on_reboot',
ENABLED: 'enabled',
ENABLE_ON_REBOOT: 'enable_on_reboot',
} as const
export type HOST_DISPLAY = (typeof HOST_DISPLAY)[keyof typeof HOST_DISPLAY]
export const HOST_SCHED_GRAN = {
CORE: 'core',
CPU: 'cpu',
SOCKET: 'socket',
} as const
export type HOST_SCHED_GRAN = (typeof HOST_SCHED_GRAN)[keyof typeof HOST_SCHED_GRAN]
export const HOST_NUMA_AFFINITY_POLICY = {
ANY: 'any',
BEST_EFFORT: 'best_effort',
DEFAULT_POLICY: 'default_policy',
} as const
export type HOST_NUMA_AFFINITY_POLICY = (typeof HOST_NUMA_AFFINITY_POLICY)[keyof typeof HOST_NUMA_AFFINITY_POLICY]
export const NETWORK_OPERATIONS = {
ATTACHING: 'attaching',
} as const
export type NETWORK_OPERATIONS = (typeof NETWORK_OPERATIONS)[keyof typeof NETWORK_OPERATIONS]
export const NETWORK_DEFAULT_LOCKING_MODE = {
DISABLED: 'disabled',
UNLOCKED: 'unlocked',
} as const
export type NETWORK_DEFAULT_LOCKING_MODE =
(typeof NETWORK_DEFAULT_LOCKING_MODE)[keyof typeof NETWORK_DEFAULT_LOCKING_MODE]
export const NETWORK_PURPOSE = {
INSECURE_NBD: 'insecure_nbd',
NBD: 'nbd',
} as const
export type NETWORK_PURPOSE = (typeof NETWORK_PURPOSE)[keyof typeof NETWORK_PURPOSE]
export const VIF_OPERATIONS = {
ATTACH: 'attach',
PLUG: 'plug',
UNPLUG: 'unplug',
} as const
export type VIF_OPERATIONS = (typeof VIF_OPERATIONS)[keyof typeof VIF_OPERATIONS]
export const VIF_LOCKING_MODE = {
DISABLED: 'disabled',
LOCKED: 'locked',
NETWORK_DEFAULT: 'network_default',
UNLOCKED: 'unlocked',
} as const
export type VIF_LOCKING_MODE = (typeof VIF_LOCKING_MODE)[keyof typeof VIF_LOCKING_MODE]
export const VIF_IPV4_CONFIGURATION_MODE = {
NONE: 'None',
STATIC: 'Static',
} as const
export type VIF_IPV4_CONFIGURATION_MODE = (typeof VIF_IPV4_CONFIGURATION_MODE)[keyof typeof VIF_IPV4_CONFIGURATION_MODE]
export const VIF_IPV6_CONFIGURATION_MODE = {
NONE: 'None',
STATIC: 'Static',
} as const
export type VIF_IPV6_CONFIGURATION_MODE = (typeof VIF_IPV6_CONFIGURATION_MODE)[keyof typeof VIF_IPV6_CONFIGURATION_MODE]
export const PIF_IGMP_STATUS = {
DISABLED: 'disabled',
ENABLED: 'enabled',
UNKNOWN: 'unknown',
} as const
export type PIF_IGMP_STATUS = (typeof PIF_IGMP_STATUS)[keyof typeof PIF_IGMP_STATUS]
export const IP_CONFIGURATION_MODE = {
DHCP: 'DHCP',
NONE: 'None',
STATIC: 'Static',
} as const
export type IP_CONFIGURATION_MODE = (typeof IP_CONFIGURATION_MODE)[keyof typeof IP_CONFIGURATION_MODE]
export const IPV6_CONFIGURATION_MODE = {
AUTOCONF: 'Autoconf',
DHCP: 'DHCP',
NONE: 'None',
STATIC: 'Static',
} as const
export type IPV6_CONFIGURATION_MODE = (typeof IPV6_CONFIGURATION_MODE)[keyof typeof IPV6_CONFIGURATION_MODE]
export const PRIMARY_ADDRESS_TYPE = {
IPV4: 'IPv4',
IPV6: 'IPv6',
} as const
export type PRIMARY_ADDRESS_TYPE = (typeof PRIMARY_ADDRESS_TYPE)[keyof typeof PRIMARY_ADDRESS_TYPE]
export const BOND_MODE = {
ACTIVE_BACKUP: 'active-backup',
BALANCE_SLB: 'balance-slb',
LACP: 'lacp',
} as const
export type BOND_MODE = (typeof BOND_MODE)[keyof typeof BOND_MODE]
export const STORAGE_OPERATIONS = {
DESTROY: 'destroy',
FORGET: 'forget',
PBD_CREATE: 'pbd_create',
PBD_DESTROY: 'pbd_destroy',
PLUG: 'plug',
SCAN: 'scan',
UNPLUG: 'unplug',
UPDATE: 'update',
VDI_CLONE: 'vdi_clone',
VDI_CREATE: 'vdi_create',
VDI_DATA_DESTROY: 'vdi_data_destroy',
VDI_DESTROY: 'vdi_destroy',
VDI_DISABLE_CBT: 'vdi_disable_cbt',
VDI_ENABLE_CBT: 'vdi_enable_cbt',
VDI_INTRODUCE: 'vdi_introduce',
VDI_LIST_CHANGED_BLOCKS: 'vdi_list_changed_blocks',
VDI_MIRROR: 'vdi_mirror',
VDI_RESIZE: 'vdi_resize',
VDI_SET_ON_BOOT: 'vdi_set_on_boot',
VDI_SNAPSHOT: 'vdi_snapshot',
} as const
export type STORAGE_OPERATIONS = (typeof STORAGE_OPERATIONS)[keyof typeof STORAGE_OPERATIONS]
export const SR_HEALTH = {
HEALTHY: 'healthy',
RECOVERING: 'recovering',
} as const
export type SR_HEALTH = (typeof SR_HEALTH)[keyof typeof SR_HEALTH]
export const VDI_OPERATIONS = {
BLOCKED: 'blocked',
CLONE: 'clone',
COPY: 'copy',
DATA_DESTROY: 'data_destroy',
DESTROY: 'destroy',
DISABLE_CBT: 'disable_cbt',
ENABLE_CBT: 'enable_cbt',
FORCE_UNLOCK: 'force_unlock',
FORGET: 'forget',
GENERATE_CONFIG: 'generate_config',
LIST_CHANGED_BLOCKS: 'list_changed_blocks',
MIRROR: 'mirror',
RESIZE: 'resize',
RESIZE_ONLINE: 'resize_online',
SET_ON_BOOT: 'set_on_boot',
SNAPSHOT: 'snapshot',
UPDATE: 'update',
} as const
export type VDI_OPERATIONS = (typeof VDI_OPERATIONS)[keyof typeof VDI_OPERATIONS]
export const VDI_TYPE = {
CBT_METADATA: 'cbt_metadata',
CRASHDUMP: 'crashdump',
EPHEMERAL: 'ephemeral',
HA_STATEFILE: 'ha_statefile',
METADATA: 'metadata',
PVS_CACHE: 'pvs_cache',
REDO_LOG: 'redo_log',
RRD: 'rrd',
SUSPEND: 'suspend',
SYSTEM: 'system',
USER: 'user',
} as const
export type VDI_TYPE = (typeof VDI_TYPE)[keyof typeof VDI_TYPE]
export const ON_BOOT = {
PERSIST: 'persist',
RESET: 'reset',
} as const
export type ON_BOOT = (typeof ON_BOOT)[keyof typeof ON_BOOT]
export const VBD_OPERATIONS = {
ATTACH: 'attach',
EJECT: 'eject',
INSERT: 'insert',
PAUSE: 'pause',
PLUG: 'plug',
UNPAUSE: 'unpause',
UNPLUG: 'unplug',
UNPLUG_FORCE: 'unplug_force',
} as const
export type VBD_OPERATIONS = (typeof VBD_OPERATIONS)[keyof typeof VBD_OPERATIONS]
export const VBD_TYPE = {
CD: 'CD',
DISK: 'Disk',
FLOPPY: 'Floppy',
} as const
export type VBD_TYPE = (typeof VBD_TYPE)[keyof typeof VBD_TYPE]
export const VBD_MODE = {
RO: 'RO',
RW: 'RW',
} as const
export type VBD_MODE = (typeof VBD_MODE)[keyof typeof VBD_MODE]
export const VTPM_OPERATIONS = {
DESTROY: 'destroy',
} as const
export type VTPM_OPERATIONS = (typeof VTPM_OPERATIONS)[keyof typeof VTPM_OPERATIONS]
export const PERSISTENCE_BACKEND = {
XAPI: 'xapi',
} as const
export type PERSISTENCE_BACKEND = (typeof PERSISTENCE_BACKEND)[keyof typeof PERSISTENCE_BACKEND]
export const CONSOLE_PROTOCOL = {
RDP: 'rdp',
RFB: 'rfb',
VT100: 'vt100',
} as const
export type CONSOLE_PROTOCOL = (typeof CONSOLE_PROTOCOL)[keyof typeof CONSOLE_PROTOCOL]
export const CLS = {
CERTIFICATE: 'Certificate',
HOST: 'Host',
POOL: 'Pool',
PVS_PROXY: 'PVS_proxy',
SR: 'SR',
VDI: 'VDI',
VM: 'VM',
VMPP: 'VMPP',
VMSS: 'VMSS',
} as const
export type CLS = (typeof CLS)[keyof typeof CLS]
export const TUNNEL_PROTOCOL = {
GRE: 'gre',
VXLAN: 'vxlan',
} as const
export type TUNNEL_PROTOCOL = (typeof TUNNEL_PROTOCOL)[keyof typeof TUNNEL_PROTOCOL]
export const SRIOV_CONFIGURATION_MODE = {
MANUAL: 'manual',
MODPROBE: 'modprobe',
SYSFS: 'sysfs',
UNKNOWN: 'unknown',
} as const
export type SRIOV_CONFIGURATION_MODE = (typeof SRIOV_CONFIGURATION_MODE)[keyof typeof SRIOV_CONFIGURATION_MODE]
export const PGPU_DOM0_ACCESS = {
DISABLED: 'disabled',
DISABLE_ON_REBOOT: 'disable_on_reboot',
ENABLED: 'enabled',
ENABLE_ON_REBOOT: 'enable_on_reboot',
} as const
export type PGPU_DOM0_ACCESS = (typeof PGPU_DOM0_ACCESS)[keyof typeof PGPU_DOM0_ACCESS]
export const ALLOCATION_ALGORITHM = {
BREADTH_FIRST: 'breadth_first',
DEPTH_FIRST: 'depth_first',
} as const
export type ALLOCATION_ALGORITHM = (typeof ALLOCATION_ALGORITHM)[keyof typeof ALLOCATION_ALGORITHM]
export const VGPU_TYPE_IMPLEMENTATION = {
GVT_G: 'gvt_g',
MXGPU: 'mxgpu',
NVIDIA: 'nvidia',
NVIDIA_SRIOV: 'nvidia_sriov',
PASSTHROUGH: 'passthrough',
} as const
export type VGPU_TYPE_IMPLEMENTATION = (typeof VGPU_TYPE_IMPLEMENTATION)[keyof typeof VGPU_TYPE_IMPLEMENTATION]
export const PVS_PROXY_STATUS = {
CACHING: 'caching',
INCOMPATIBLE_PROTOCOL_VERSION: 'incompatible_protocol_version',
INCOMPATIBLE_WRITE_CACHE_MODE: 'incompatible_write_cache_mode',
INITIALISED: 'initialised',
STOPPED: 'stopped',
} as const
export type PVS_PROXY_STATUS = (typeof PVS_PROXY_STATUS)[keyof typeof PVS_PROXY_STATUS]
export const SDN_CONTROLLER_PROTOCOL = {
PSSL: 'pssl',
SSL: 'ssl',
} as const
export type SDN_CONTROLLER_PROTOCOL = (typeof SDN_CONTROLLER_PROTOCOL)[keyof typeof SDN_CONTROLLER_PROTOCOL]
export const VUSB_OPERATIONS = {
ATTACH: 'attach',
PLUG: 'plug',
UNPLUG: 'unplug',
} as const
export type VUSB_OPERATIONS = (typeof VUSB_OPERATIONS)[keyof typeof VUSB_OPERATIONS]
export const CLUSTER_OPERATION = {
ADD: 'add',
DESTROY: 'destroy',
DISABLE: 'disable',
ENABLE: 'enable',
REMOVE: 'remove',
} as const
export type CLUSTER_OPERATION = (typeof CLUSTER_OPERATION)[keyof typeof CLUSTER_OPERATION]
export const CLUSTER_HOST_OPERATION = {
DESTROY: 'destroy',
DISABLE: 'disable',
ENABLE: 'enable',
} as const
export type CLUSTER_HOST_OPERATION = (typeof CLUSTER_HOST_OPERATION)[keyof typeof CLUSTER_HOST_OPERATION]
export const CERTIFICATE_TYPE = {
CA: 'ca',
HOST: 'host',
HOST_INTERNAL: 'host_internal',
} as const
export type CERTIFICATE_TYPE = (typeof CERTIFICATE_TYPE)[keyof typeof CERTIFICATE_TYPE]
export const OPAQUE_REF = { EMPTY: 'OpaqueRef:NULL' } as const
export type OPAQUE_REF_NULL = (typeof OPAQUE_REF)['EMPTY']
export const BACKUP_TYPE = { backup: 'backup', metadata: 'metadataBackup', mirror: 'mirrorBackup' } as const
export type BACKUP_TYPE = (typeof BACKUP_TYPE)[keyof typeof BACKUP_TYPE]
export const SUPPORTED_VDI_FORMAT = {
raw: 'raw',
vhd: 'vhd',
qcow2: 'qcow2',
} as const
export type SUPPORTED_VDI_FORMAT = (typeof SUPPORTED_VDI_FORMAT)[keyof typeof SUPPORTED_VDI_FORMAT]
// ----- XAPI Stats
type XapiStatsResponse<T> = {
endTimestamp: number
interval: number
stats: T
}
type StatValues = (number | null)[]
type RecordStatValues = Record<string, StatValues>
export type XapiStatsGranularity = 'seconds' | 'minutes' | 'hours' | 'days'
export type XapiHostStatsRaw = {
cpus?: RecordStatValues
ioThroughput?: {
r: RecordStatValues
w: RecordStatValues
}
iops?: {
r: RecordStatValues
w: RecordStatValues
}
iowait?: RecordStatValues
latency?: {
r: RecordStatValues
w: RecordStatValues
}
load?: StatValues
memory?: StatValues
memoryFree?: StatValues
pifs?: {
rx: RecordStatValues
tx: RecordStatValues
}
}
export type XapiHostStats = XapiStatsResponse<XapiHostStatsRaw>
export type XapiVmStatsRaw = {
cpus?: RecordStatValues
cpuUsage?: StatValues
runstateFullrun?: StatValues
runstateFullContention?: StatValues
runstatePartialRun?: StatValues
runstatePartialContention?: StatValues
runstateConcurrencyHazard?: StatValues
runstateBlocked?: StatValues
iops?: {
r: RecordStatValues
w: RecordStatValues
}
memory?: StatValues
memoryFree?: StatValues
memoryTarget?: StatValues
vifs?: {
rx: RecordStatValues
tx: RecordStatValues
}
vifErrors?: {
rx: RecordStatValues
tx: RecordStatValues
}
xvds?: {
w?: RecordStatValues
r?: RecordStatValues
total?: RecordStatValues
}
vbdLatency?: {
w: RecordStatValues
r: RecordStatValues
}
vbdIowait?: RecordStatValues
vbdInflight?: RecordStatValues
vbdAvgquSz?: RecordStatValues
}
export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>

View File

@@ -1,6 +0,0 @@
export * from './common.mjs'
export * from './xen-api.mjs'
export * from './xo.mjs'
export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/complex-matcher.mjs'

View File

@@ -1,5 +0,0 @@
export type Or<T> = Id<{ __or: T[] }>
export type Id<T> = { id: T }
export type IdOr<T> = Id<T> | Or<T>

View File

@@ -1,21 +0,0 @@
import { Branded } from '../common.mjs'
export interface VatesTask {
_abortController: AbortController
_onProgress(data?: unknown): void
id: Branded<'vates-task'>
status: 'failure' | 'pending' | 'success'
abort(reason?: unknown): void
failure(error?: unknown): void
info(message: string, data: unknown): void
run<T>(fn: () => T | (() => Promise<T>)): Promise<T>
runInside<T>(fn: () => T | (() => Promise<T>)): Promise<T>
set(name: string, value: unknown): void
start(): void
success(result?: unknown): void
warning(message: string, data: unknown): void
wrap<T>(fn: () => T | (() => Promise<T>)): Promise<T>
wrapInside<T>(fn: () => T | (() => Promise<T>)): Promise<T>
}

View File

@@ -1,331 +0,0 @@
import {
WrappedXenApiRecord,
XenApiHost,
XenApiHostWrapped,
XenApiNetwork,
XenApiNetworkWrapped,
XenApiRecord,
XenApiSr,
XenApiTask,
XenApiVbd,
XenApiVdi,
XenApiVif,
XenApiVm,
XenApiVmWrapped,
XenApiVtpm,
} from '../xen-api.mjs'
import type {
OPAQUE_REF_NULL,
SUPPORTED_VDI_FORMAT,
VBD_MODE,
VBD_TYPE,
VDI_TYPE,
VIF_LOCKING_MODE,
} from '../common.mjs'
import type { PassThrough, Readable } from 'node:stream'
import type {
XoGpuGroup,
XoHost,
XoNetwork,
XoPif,
XoSr,
XoUser,
XoVdi,
XoVgpuType,
XoVm,
XoVmTemplate,
XoVif,
} from '../xo.mjs'
export type XcpPatches = {
changelog?: {
author: string
date: number
description: string
}
description: string
license: string
name: string
release: string
size: number
url: string
version: string
}
export type XsPatches = {
conflicts?: string[]
date: string
description: string
documentationUrl?: string
guidances: string
name: string
id?: string
paid?: boolean
requirements?: string[]
upgrade?: boolean
url?: string
uuid?: string
}
export interface Xapi {
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
getField<T extends XenApiRecord, K extends keyof T>(
type: Extract<WrappedXenApiRecord, T>['$type'],
ref: T['$ref'],
field: K
): Promise<T[K]>
createNetwork(
params:
| {
name: string
description?: string
mtu?: number
}
| {
name: string
description?: string
pifId: XoPif['id']
mtu?: number
/* between 0 and 4094 */
vlan: number
}
): Promise<XenApiNetworkWrapped>
deleteNetwork(id: XoNetwork['id']): Promise<void>
deleteVif(vifId: XoVif['id']): Promise<void>
exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
migrateVm(
vmId: XoVm['id'],
hostXapi: Xapi,
hostId: XoHost['id'],
opts?: {
force?: boolean
mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
migrationNetworkId?: XoNetwork['id']
sr?: XoSr['id']
bypassAssert?: boolean
}
): Promise<void>
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
pool_emergencyShutdown(): Promise<void>
resumeVm(id: XoVm['id']): Promise<void>
unpauseVm(id: XoVm['id']): Promise<void>
cloneVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
fast?: boolean
}
): Promise<XenApiVmWrapped>
copyVm(
vmId: XoVm['id'],
opts?: {
nameLabel?: string
srOrSrId?: XoSr['id'] | XoSr
}
): Promise<XenApiVmWrapped>
remoteCopyVm(
vmId: XoVm['id'],
targetXapi: Xapi,
targetSrId: XoSr['id'],
opts?: {
compress?: 'zstd' | 'gzip' | boolean
nameLabel?: string
}
): Promise<{ vm: XenApiVmWrapped }>
forgetSr(id: XoSr['id']): Promise<void>
SR_importVdi(
ref: XenApiSr['$ref'],
stream: Readable,
opts?: {
format?: SUPPORTED_VDI_FORMAT
name_description?: XoVdi['name_description']
name_label?: XoVdi['name_label']
other_config?: XoVdi['other_config']
read_only?: boolean
sharable?: boolean
SR?: XenApiSr['$ref']
tags?: XoVdi['tags']
type?: XoVdi['type']
virtual_size?: XoVdi['size']
xenstore_data?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
startVm(
id: XoVm['id'],
opts?: {
bypassMacAddressesCheck?: boolean
force?: boolean
hostId?: XoHost['id']
/**
* if startOnly is true and the VM is not halted, throw VM_BAD_POWER_STATE
* otherwise, unpause/resume the VM
*
* @default false
*/
startOnly?: boolean
}
): Promise<void>
VM_export(
vmRef: XenApiVm['$ref'],
opts?: { cancelToken?: unknown; compress?: boolean; useSnapshot?: boolean }
): ReturnType<Xapi['getResource']>
VM_import(
stream: Readable,
srRef?: XenApiSr['$ref'],
onVmCreation?: null | ((vm: XenApiVm) => unknown)
): Promise<XenApiVm['$ref']>
createVm(
templateUuid: XoVmTemplate['uuid'],
metadataVm: {
affinityHost?: XoHost['id']
autoPoweron?: boolean
cpus?: number
memory?: number
name_description?: string
name_label: string
nameLabel?: string
clone?: boolean
installRepository?: XoVdi['id'] | '' | null
vdis?: (
| /** Create VDI */ {
name_label: string
size: number
sr?: XoSr['id']
name_description?: string
}
| /** Update VDI*/ {
userdevice: string
name_label?: string
size?: number
sr?: XoSr['id']
name_description?: string
}
| /** Destroy VDI*/ { destroy: true; userdevice: string }
)[]
vifs?: (
| /** Create/update VIF */ {
device?: string
ipv4_allowed?: string[]
ipv6_allowed?: string[]
mac?: string
mtu?: number
network: string
}
| /** Destroy VIF */ { destroy: true; device: string }
)[]
existingVdis?: {
$SR: XoSr['id']
size: number
userdevice: string
}[]
vgpuType?: XoVgpuType['id']
gpuGroup?: XoGpuGroup['id']
copyHostBiosStrings?: boolean
hvmBootFirmware?: 'uefi' | 'bios'
secureBoot?: boolean
},
checkLimits?: boolean,
creatorId?: XoUser['id'],
opts?: { destroyAllVifs: boolean }
): Promise<XenApiVmWrapped>
moveVdi(
vdiId: XenApiVdi['$ref'] | XoVdi['id'],
srId: XenApiSr['$ref'] | XoSr['id'],
opts?: { _failOnCbtError?: boolean }
): Promise<XenApiVdi>
VBD_create(params: {
bootable?: boolean
empty?: boolean
mode?: VBD_MODE
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
type?: VBD_TYPE
unpluggable?: boolean
userdevice?: string
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
VM: XenApiVm['$ref']
}): Promise<XenApiVbd['$ref']>
VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
VDI_create(
options: {
name_description?: XoVdi['name_description']
name_label?: XoVdi['name_label']
other_config?: XoVdi['other_config']
read_only?: boolean
sharable?: boolean
SR?: XenApiSr['$ref']
tags?: XoVdi['tags']
type?: VDI_TYPE
virtual_size: XoVdi['size']
xenstore_data?: Record<string, string>
},
extraOptions?: {
sm_config?: Record<string, string>
}
): Promise<XenApiVdi['$ref']>
VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
VDI_exportContent(
vdiRef: XenApiVdi['$ref'],
opts: { baseRef?: string; cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
): Promise<Readable & { length?: number }>
VDI_importContent(
vdiRef: XenApiVdi['$ref'],
stream: Readable,
opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
): Promise<void>
VIF_create(
options: {
currently_attached?: boolean
device?: string
ipv4_allowed?: string[]
ipv6_allowed?: string[]
locking_mode?: VIF_LOCKING_MODE
MTU?: number
network: XenApiNetwork['$ref']
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
VM: XenApiVm['$ref']
},
extraOptions?: {
MAC?: string
}
): Promise<XenApiVif['$ref']>
VM_createCloudInitConfig(
vmRef: XenApiVm['$ref'],
cloudConfig: string,
opts?: { networkConfig?: string }
): Promise<XoVdi['uuid']>
VM_destroy(
vmRef: XenApiVm['$ref'],
opts?: {
deleteDisks?: boolean
force?: boolean
bypassBlockedOperation?: boolean
forceDeleteDefaultTemplate?: boolean
}
): Promise<void>
getResource(
pathname: string,
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
): Promise<{ body: Readable }>
clearHost(host: Pick<XenApiHostWrapped, '$ref' | '$pool'>, force?: boolean): Promise<void>
disableHost(hostId: XoHost['id']): Promise<void>
enableHost(hostId: XoHost['id']): Promise<void>
getRecordByUuid<
Type extends WrappedXenApiRecord['$type'],
XenApiRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: Type }>,
>(
type: Type,
uuid: XenApiRecord['uuid']
): Promise<XenApiRecord>
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,257 +0,0 @@
import { EventEmitter } from 'node:stream'
import type {
AnyXoBackupJob,
AnyXoJob,
AnyXoLog,
XapiXoRecord,
XoAuthenticationToken,
XoBackupRepository,
XoConfigBackupArchive,
XoGroup,
XoHost,
XoPool,
XoPoolBackupArchive,
XoProxy,
XoSchedule,
XoServer,
XoTask,
XoUser,
XoVm,
XoVmBackupArchive,
} from './xo.mjs'
import { VatesTask } from './lib/vates-task.mjs'
import {
Xapi,
XapiHostStats,
XapiPoolStats,
XapiStatsGranularity,
XapiVmStats,
XenApiGpuGroupWrapped,
XenApiHostWrapped,
XenApiMessageWrapped,
XenApiNetworkWrapped,
XenApiPbdWrapped,
XenApiPciWrapped,
XenApiPgpuWrapped,
XenApiPifWrapped,
XenApiPoolWrapped,
XenApiSmWrapped,
XenApiSrWrapped,
XenApiVbdWrapped,
XenApiVdiWrapped,
XenApiVgpuTypeWrapped,
XenApiVgpuWrapped,
XenApiVifWrapped,
XenApiVmWrapped,
XenApiVtpmWrapped,
} from './index.mjs'
export type XapiConnection = Xapi & {
status: string
pool?: { uuid: string }
sessionId: string
_url?: { protocol: string; hostname: string; port?: string }
}
type XapiRecordByXapiXoRecord = {
gpuGroup: XenApiGpuGroupWrapped
host: XenApiHostWrapped
message: XenApiMessageWrapped
network: XenApiNetworkWrapped
PBD: XenApiPbdWrapped
PCI: XenApiPciWrapped
PGPU: XenApiPgpuWrapped
PIF: XenApiPifWrapped
pool: XenApiPoolWrapped
SR: XenApiSrWrapped
SM: XenApiSmWrapped
VBD: XenApiVbdWrapped
VDI: XenApiVdiWrapped
'VDI-snapshot': XenApiVdiWrapped
'VDI-unmanaged': XenApiVdiWrapped
vgpu: XenApiVgpuWrapped
vgpuType: XenApiVgpuTypeWrapped
VIF: XenApiVifWrapped
VM: XenApiVmWrapped
'VM-controller': XenApiVmWrapped
'VM-snapshot': XenApiVmWrapped
'VM-template': XenApiVmWrapped
VTPM: XenApiVtpmWrapped
}
export type XoApp = {
config: {
get<T = string>(path: string): T
getOptional(path: string): Record<string, string> | undefined
getOptionalDuration(path: string): number | undefined
getGuiRoutes(): Promise<{
default: {
url: string
path: string
}
v5?: {
url: string
path: string
}
v6?: {
url: string
path: string
}
[key: string]:
| undefined
| {
url: string
path: string
}
}>
}
objects: EventEmitter & {
allIndexes: {
type: {
getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
}
}
}
tasks: EventEmitter & {
abort(id: XoTask['id'], reason?: string): Promise<void>
clearLogs(): Promise<void>
create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
deleteLog(id: XoTask['id']): Promise<void>
get(id: XoTask['id']): Promise<XoTask>
list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
}
apiContext: {
user?: XoUser
permission?: XoUser['permission'] | 'none' | null
}
// methods ------------
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
authenticateUser: (
credentials: { token?: string; username?: string; password?: string },
userData?: { ip?: string },
opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
/* Throw if no authorization */
checkFeatureAuthorization(featureCode: string): Promise<void>
/* connect a server (XCP-ng/XenServer) */
connectXenServer(id: XoServer['id']): Promise<void>
createAuthenticationToken(opts: {
client?: {
id?: string
[key: string]: unknown
}
description?: string
expiresIn?: string | number
userId: XoUser['id']
}): Promise<XoAuthenticationToken>
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
/* disconnect a server (XCP-ng/XenServer) */
disconnectXenServer(id: XoServer['id']): Promise<void>
getAllGroups(): Promise<XoGroup[]>
getAllProxies(): Promise<XoProxy[]>
getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
getAllJobs(type?: string): Promise<AnyXoJob[]>
getProxy(id: XoProxy['id']): Promise<XoProxy>
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
getAllRemotes(): Promise<XoBackupRepository[]>
getAllRemotesInfo(): Promise<
Record<
XoBackupRepository['id'],
{
size?: number
used?: number
available?: number
encryption?: {
algorithm: string
isLegacy: boolean
recommanded: string
}
}
>
>
getAllSchedules(): Promise<XoSchedule[]>
getAllUsers(): Promise<XoUser[]>
getAllXenServers(): Promise<XoServer[]>
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
getBackupNgLogsSorted(opts: {
after?: number
before?: number
filter: (log: AnyXoLog) => boolean
limit?: number
}): Promise<AnyXoLog[]>
getGroup(id: XoGroup['id']): Promise<XoGroup>
getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
getObjectsByType: <T extends XapiXoRecord>(
type: T['type'],
opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
) => Record<T['id'], T> | undefined
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
getUser: (id: XoUser['id']) => Promise<XoUser>
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
getXapiObject: <T extends XapiXoRecord>(
maybeId: T['id'] | T,
type: T['type'] | T['type'][]
) => XapiRecordByXapiXoRecord[T['type']]
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
getXenServer(id: XoServer['id']): Promise<XoServer>
hasFeatureAuthorization(featureCode: string): Promise<boolean>
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
}>
listVmBackupsNg(
backupRepositoryIds: XoBackupRepository['id'][],
opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
/** Allow to add a new server in the DB (XCP-ng/XenServer) */
registerXenServer(
body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
allowUnauthorized?: XoServer['allowUnauthorized']
password: string
readOnly?: XoServer['readOnly']
}
): Promise<XoServer>
rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise<void>
rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise<void>
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
runJob(job: AnyXoJob, schedule: XoSchedule): void
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
/** Remove a server from the DB (XCP-ng/XenServer) */
unregisterXenServer(id: XoServer['id']): Promise<void>
updateUser(
id: XoUser['id'],
updates: {
/**
* @deprecated
*/
email?: string
authProviders?: Record<string, string>
name?: string
password?: string
permission?: string
preferences?: Record<string, string>
}
): Promise<void>
updateGroup(
id: XoGroup['id'],
updates: {
name?: string
}
): void
getAllXapis(): Record<string, XapiConnection>
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
}

View File

@@ -1,875 +0,0 @@
// Types based on xapi-object-to-xo
import type {
Branded,
DOMAIN_TYPE,
HOST_ALLOWED_OPERATIONS,
HOST_POWER_STATE,
IP_CONFIGURATION_MODE,
IPV6_CONFIGURATION_MODE,
NETWORK_OPERATIONS,
PGPU_DOM0_ACCESS,
POOL_ALLOWED_OPERATIONS,
PRIMARY_ADDRESS_TYPE,
STORAGE_OPERATIONS,
VDI_OPERATIONS,
VDI_TYPE,
VIF_LOCKING_MODE,
VM_OPERATIONS,
VM_POWER_STATE,
} from './common.mjs'
import type * as CMType from './lib/complex-matcher.mjs'
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
type BaseXapiXo = {
$pool: XoPool['id']
/**
* @deprecated use $pool instead
*/
$poolId: XoPool['id']
_xapiRef: string
uuid: string
}
type BaseXoVm = BaseXapiXo & {
$VBDs: XoVbd['id'][]
$VGPUs: XoVgpu['id'][]
$container: XoPool['id'] | XoHost['id']
CPUs: {
max: number
number: number
}
PV_args?: string
VGPUs: XoVgpu['id'][]
VIFs: XoVif['id'][]
VTPMs: XoVtpm['id'][]
addresses: Record<string, string>
affinityHost?: XoHost['id']
attachedPcis?: string[]
auto_poweron: boolean
bios_strings: Record<string, string>
blockedOperations: Record<VM_OPERATIONS, string>
boot: Record<string, string>
coresPerSocket?: number
cpuCap?: number
cpuMask?: number[]
cpuWeight?: number
creation: Record<string, string>
current_operations: Record<string, VM_OPERATIONS>
docker?: {
containers?: string[]
enabled: boolean
info?: string
/** @deprecated */
process?: string
version?: string
}
/**
* @deprecated use isNestedVirtEnabled instead
*/
expNestedHvm: boolean
isNestedVirtEnabled: boolean
hasVendorDevice: boolean
high_availability: string
installTime?: number | null
isFirmwareSupported: boolean
memory: {
dynamic: number[]
size: number
static: number[]
usage?: number
}
mainIpAddress?: string
managementAgentDetected?: boolean
name_description: string
name_label: string
needsVtpm: boolean
nicType?: string
notes?: string
os_version: Record<string, string> | null
other: Record<string, string>
parent?: XoVm['id']
power_state: VM_POWER_STATE
pvDriversDetected?: boolean
pvDriversUpToDate?: boolean
pvDriversVersion?: string
resourceSet?: string
secureBoot: boolean
snapshots: XoVmSnapshot['id'][]
startDelay: number
startTime?: number | null
suspendSr?: XoSr['id']
tags: string[]
vga?: string
videoram?: number
viridian: boolean
virtualizationMode: DOMAIN_TYPE
xenStoreData: Record<string, string>
/**
* @deprecated use pvDriversVersion instead
*/
xentools?:
| false
| {
major: null | number
minor: null | number
version: null | string
}
}
export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
body: {
value?: string
name: string
}
object: {
type: XapiXoRecord['type'] | 'unknown'
uuid: XapiXoRecord['uuid']
href?: string
}
}
// TODO: to be typed when Bastien.N has finished working on the XO task
type BaseXoLog = {
id: Branded<'xo-log'>
infos?: { data: unknown; message: string }[]
[key: string]: unknown
}
export type XoBackupLog = BaseXoLog & {
status: 'success' | 'skipped' | 'interrupted' | 'failure' | 'pending'
message: 'backup'
start: number
end?: number
tasks?: XoTask[]
jobId?: AnyXoBackupJob['id']
}
export type XoRestoreLog = BaseXoLog & {
message: 'restore'
}
export type XoVmBackupArchive = {
id: Branded<'vm-backup-archive'>
type: 'xo-vm-backup'
backupRepository: XoBackupRepository['id']
disks: { id: string; name: string; uuid: XoVdiSnapshot['uuid'] }[]
isImmutable?: boolean
jobId: XoVmBackupJob['id']
mode: XoVmBackupJob['mode']
scheduleId: XoSchedule['id']
size: number
timestamp: number
vm: {
uuid: XoVm['uuid']
name_description: string
name_label: string
tags: XoVm['tags']
}
differencingVhds?: number
dynamicVhds?: number
withMemory: boolean
}
type XoMetadataBackupArchive = {
id: Branded<'metadata-backup-archive'>
backupRepository: XoBackupRepository['id']
jobId: XoVmBackupJob['id']
jobName: XoVmBackupJob['name']
scheduleId: XoSchedule['id']
scheduleName: XoSchedule['name']
timestamp: number
}
export type XoConfigBackupArchive = XoMetadataBackupArchive & {
id: Branded<'config-backup-archive'>
data: string
type: 'xo-config-backup'
}
export type XoPoolBackupArchive = XoMetadataBackupArchive & {
id: Branded<'pool-backup-archive'>
pool: XenApiPool
poolMaster: XenApiHost
type: 'xo-pool-metadata-backup'
}
export type XoBackupRepository = {
benchmarks?: { readRate: number; timestamp: number; writeRate: number }[]
enabled: boolean
error?: Record<string, unknown>
id: Branded<'backup-repository'>
name: string
options?: string
proxy?: XoProxy['id']
url: string
}
export type XoGpuGroup = BaseXapiXo & {
id: Branded<'gpu-group'>
type: 'gpuGroup'
}
export type XoGroup = {
id: Branded<'group'>
name: string
provider?: string
providerGroupId?: string
users: XoUser['id'][]
}
export type XoHost = BaseXapiXo & {
$PBDs: XoPbd['id'][]
$PCIs: XoPci['id'][]
$PGPUs: XoPgpu['id'][]
$PIFs: XoPif['id'][]
PCIs: XoPci['id'][]
PGPUs: XoPgpu['id'][]
PIFs: XoPif['id'][]
/**
* @deprecated use cpus instead
*/
CPUs: Record<string, string>
address: string
agentStartTime: null | number
bios_strings: Record<string, string>
build: string
certificates?: {
fingerprint: string
notAfter: number
}[]
chipset_info: {
iommu?: boolean
}
controlDomain?: XoVmController['id']
cpus: {
cores?: number
sockets?: number
}
current_operations: Record<string, HOST_ALLOWED_OPERATIONS>
enabled: boolean
hostname: string
hvmCapable: boolean
id: Branded<'host'>
iscsiIqn: string
license_expiry: null | number
license_params: Record<string, string>
license_server: Record<string, string>
logging: Record<string, string>
memory: {
size: number
/**
* @deprecated
*/
total?: number
usage: number
}
multipathing: boolean
name_description: string
name_label: string
otherConfig: Record<string, string>
/**
* @deprecated
*/
patches: XoHostPatch['id'][]
power_state: HOST_POWER_STATE
powerOnMode: string
productBrand: string
rebootRequired: boolean
residentVms: XoVm['id'][]
startTime: null | number
supplementalPacks:
| {
author: string
description: string
guidance: string
hosts: XoHost['id'][]
name: string
size: number
version: string
vdi: XoVdi['id']
}[]
| {
author: string
description: string
name: string
version: string
}[]
tags: string[]
type: 'host'
version: string
zstdSupported: boolean
}
export type XoHostPatch = BaseXapiXo & {
id: Branded<'host_patch'>
type: 'host_patch'
}
export type XoMessage = BaseXapiXo & {
$object: XapiXoRecord['id']
body: string
id: Branded<'message'>
name: string
time: number
type: 'message'
}
export type XoNetwork = BaseXapiXo & {
MTU: number
PIFs: XoPif['id'][]
VIFs: XoVif['id'][]
automatic: boolean
bridge: string
current_operations: Record<string, NETWORK_OPERATIONS>
defaultIsLocked: boolean
id: Branded<'network'>
insecureNbd?: boolean
name_description: string
name_label: string
nbd?: boolean
other_config: Record<string, string>
tags: string[]
type: 'network'
}
export type XoPbd = BaseXapiXo & {
attached: boolean
device_config:
| { device: string }
| { location: string }
| { path: string; location: string; legacy_mode: string }
| { provisioning: string; redundancy: string; 'group-name': string }
| { server: string; serverpath: string }
| { type: string; location: string }
| Record<string, string>
host: XoHost['id']
id: Branded<'PBD'>
otherConfig: Record<string, string>
SR: XoSr['id']
type: 'PBD'
}
export type XoPci = BaseXapiXo & {
$host?: XoHost['id']
class_name: string
device_name: string
id: Branded<'PCI'>
pci_id: string
type: 'PCI'
}
export type XoPgpu = BaseXapiXo & {
$host: XoHost['id']
$vgpus: XoVgpu['id'][]
dom0Access: PGPU_DOM0_ACCESS
enabledVgpuTypes: XoVgpuType['id'][]
gpuGroup?: XoGpuGroup['id']
host: XoHost['id']
id: Branded<'PGPU'>
isSystemDisplayDevice: boolean
pci?: XoPci['id']
// it seems that supportedVgpuMaxCapcities always return undefined (See: xapi-objet-to-xo)
/**
* @deprecated
*/
supportedVgpuMaxCapcities?: never
supportedVgpuTypes: XoVgpuType['id'][]
type: 'PGPU'
vgpus: XoVgpu['id'][]
}
export type XoPif = BaseXapiXo & {
$host: XoHost['id']
$network: XoNetwork['id']
attached: boolean
bondMaster?: XoPif['id']
bondSlaves?: XoPif['id'][]
carrier: boolean
device: string
deviceName?: string
disallowUnplug: boolean
dns: string
gateway: string
id: Branded<'PIF'>
ip: string
ipv6: string[]
ipv6Mode: IPV6_CONFIGURATION_MODE
isBondMaster: boolean
isBondSlave: boolean
mac: string
management: boolean
mode: IP_CONFIGURATION_MODE
mtu: number
netmask: string
physical: boolean
primaryAddressType: PRIMARY_ADDRESS_TYPE
speed?: number
type: 'PIF'
vlan: number
}
export type XoPool = BaseXapiXo & {
auto_poweron: boolean
cpus: {
cores?: number
sockets?: number
}
crashDumpSr?: XoSr['id']
current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
default_SR?: XoSr['id']
HA_enabled: boolean
haSrs: XoSr['id'][]
id: Branded<'pool'>
master: XoHost['id']
migrationCompression?: boolean
name_description: string
name_label: string
otherConfig: Record<string, string>
platform_version: string
suspendSr?: XoSr['id']
tags: string[]
type: 'pool'
vtpmSupported: boolean
zstdSupported: boolean
}
export type XoProxy = {
id: Branded<'proxy'>
url: string
version?: string
name: string
} & (
| { address?: undefined; vmUuid: XoVm['id'] }
| { address: string; vmUuid?: undefined }
| { address: string; vmUuid: XoVm['id'] }
)
type BaseXoJob = {
id: Branded<'job'>
name?: string
}
type XoBackupJobGeneralSettings = {
backupReportTpl?: 'compactMjml' | 'mjml'
reportWhen?: 'always' | 'error' | 'failure' | 'never'
reportRecipients?: string[]
hideSuccessfulItems?: boolean
[key: string]: unknown
}
export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
cbtDestroySnapshotData?: boolean
concurrency?: number
longTermRetention?: {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
checkpointSnapshot?: boolean
offlineSnapshot?: boolean
maxExportRate?: number
nbdConcurrency?: number
nRetriesVmBackupFailures?: number
preferNbd?: boolean
timezone?: string
mergeBackupsSynchronously?: boolean
offlineBackup?: boolean
timeout?: number
}
export type XoVmBackupJobScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags']
fullInterval?: number
copyRetention?: number
snapshotRetention?: number
cbtDestroySnapshotData?: boolean
healthCheckSr?: XoSr['id']
[key: string]: unknown
}
export type XoVmBackupJob = BaseXoJob & {
compression?: 'native' | 'zstd' | ''
proxy?: XoProxy['id']
mode: 'full' | 'delta'
remotes?: CMType.IdOr<XoBackupRepository['id']>
vms: CMType.IdOr<XoVm['id']> | Record<string, unknown>
srs?: CMType.IdOr<XoSr['id']>
type: 'backup'
settings: {
'': XoVmBackupJobGeneralSettings
[key: XoSchedule['id']]: XoVmBackupJobScheduleSettings | undefined
}
}
export type XoMetadataBackupJobGeneralSettings = XoBackupJobGeneralSettings
export type XoMetadataBackupJobScheduleSettings = {
retentionPoolMetadata?: number
retentionXoMetadata?: number
[key: string]: unknown
}
export type XoMetadataBackupJob = BaseXoJob & {
type: 'metadataBackup'
pools?: CMType.IdOr<XoPool['id']>
remotes: CMType.IdOr<XoBackupRepository['id']>
settings: {
''?: XoMetadataBackupJobGeneralSettings
[scheduleId: XoSchedule['id']]: XoMetadataBackupJobScheduleSettings | undefined
}
xoMetadata?: boolean
userId: XoUser['id']
proxy?: XoProxy['id']
}
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
concurrency?: number
nRetriesVmBackupFailures?: number
mergeBackupsSynchronously?: boolean
timeout?: number
maxExportRate?: number
backupReportTpl?: 'compactMjml'
reportWhen: 'failure'
}
export type XoMirrorBackupScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags']
healthCheckSr?: XoSr['id']
[key: string]: unknown
}
export type XoMirrorBackupJob = BaseXoJob & {
type: 'mirrorBackup'
mode: 'full' | 'delta'
sourceRemote: XoBackupRepository['id']
remotes: CMType.IdOr<XoBackupRepository['id']>
proxy?: XoProxy['id']
settings: {
'': XoMirrorBackupGeneralSettings
[scheduleId: XoSchedule['id']]: XoMirrorBackupScheduleSettings | undefined
}
}
export type XoJob = BaseXoJob & {
type: 'call'
}
export type XoSchedule = {
cron: string
enabled: boolean
id: Branded<'schedule'>
jobId: AnyXoJob['id']
name?: string
timezone?: string
}
export type XoServer = {
allowUnauthorized: boolean
enabled: boolean
error?: Record<string, unknown>
host: string
httpProxy?: string
id: Branded<'server'>
label?: string
master?: XoHost['id']
poolId?: XoPool['id']
poolNameDescription?: string
poolNameLabel?: string
readOnly: boolean
status: 'connected' | 'disconnected' | 'connecting'
username: string
}
export type XoSr = BaseXapiXo & {
$PBDs: XoPbd['id'][]
$container: XoPool['id'] | XoHost['id']
VDIs: AnyXoVdi['id'][]
allocationStrategy?: 'thin' | 'thick' | 'unknown'
content_type: string
current_operations: Record<string, STORAGE_OPERATIONS>
id: Branded<'SR'>
inMaintenanceMode: boolean
name_description: string
name_label: string
other_config: Record<string, string>
physical_usage: number
shared: boolean
size: number
sm_config: Record<string, string>
SR_type: string
tags: string[]
type: 'SR'
usage: number
}
export type XoSm = BaseXapiXo & {
id: Branded<'SM'>
SM_type: string
vendor: string
name_label: string
name_description: string
configuration: Record<string, string>
features: Record<string, number>
driver_filename: string
required_cluster_stack: string[]
supported_image_formats: string[]
type: 'SM'
}
export type XoTask = {
abortionRequestedAt?: number
data?: Record<string, string>
end?: number
id: Branded<'task'>
infos?: { data: unknown; message: string }[]
progress?: number
properties: {
method?: string
name?: string
objectId?: string
objectType?:
| XapiXoRecord['type']
| 'backup'
| 'backup-archive'
| 'backup-job'
| 'backup-log'
| 'backup-repository'
| 'group'
| 'proxy'
| 'restore'
| 'restore-log'
| 'schedule'
| 'server'
| 'task'
| 'user'
params?: Record<string, unknown>
progress?: number
type?: string
userId?: string
[key: string]: unknown | undefined
}
result: Record<string, unknown>
start: number
status: 'failure' | 'interrupted' | 'pending' | 'success'
tasks?: XoTask[]
updatedAt?: number
warnings?: { data: unknown; message: string }[]
}
export type XoUser = {
authProviders?: Record<string, string>
email: string
groups: XoGroup['id'][]
id: Branded<'user'>
name?: string
permission: string
pw_hash?: string
preferences: Record<string, string>
}
export type XoAuthenticationToken = {
client?: {
id: string
[key: string]: unknown
}
created_at?: number
description?: string
user_id: XoUser['id']
expiration: number
last_uses?: Record<string, { timestamp: number }>
id: Branded<'authentication-token'>
}
export type XoVbd = BaseXapiXo & {
attached: boolean
bootable: boolean
device: string | null
id: Branded<'VBD'>
is_cd_drive: boolean
position: string
read_only: boolean
type: 'VBD'
VDI?: XoVdi['id'] | XoVdiSnapshot['id']
VM: AnyXoVm['id']
}
type BaseXoVdi = BaseXapiXo & {
$SR: XoSr['id']
$VBDs: XoVbd['id'][]
VDI_type: VDI_TYPE
cbt_enabled?: boolean
current_operations: Record<string, VDI_OPERATIONS>
missing: boolean
name_description: string
name_label: string
other_config: Record<string, string>
parent?: XoVdiUnmanaged['id']
image_format?: string
size: number
snapshots: XoVdiSnapshot['id'][]
tags: string[]
usage: number
}
export type XoVdi = BaseXoVdi & {
id: Branded<'VDI'>
type: 'VDI'
}
export type XoVdiSnapshot = BaseXoVdi & {
id: Branded<'VDI-snapshot'>
snapshot_time: number
$snapshot_of?: XoVdi['id']
type: 'VDI-snapshot'
}
export type XoVdiUnmanaged = Omit<BaseXoVdi, '$VBDs'> & {
id: Branded<'VDI-unmanaged'>
type: 'VDI-unmanaged'
/**
* Unmanaged VDI have no VBDs
*/
$VBDs: never[]
}
export type XoVgpu = BaseXapiXo & {
id: Branded<'VGPU'>
type: 'vgpu'
}
export type XoVgpuType = BaseXapiXo & {
id: Branded<'vgpu-type'>
type: 'vgpuType'
}
export type XoVif = BaseXapiXo & {
$VM: XoVm['id']
$network: XoNetwork['id']
allowedIpv4Addresses: string[]
allowedIpv6Addresses: string[]
attached: boolean
device: string
id: Branded<'VIF'>
lockingMode: VIF_LOCKING_MODE
MAC: string
MTU: number
other_config: Record<string, string>
/**
* In kB/s
*/
rateLimit?: number
txChecksumming: boolean
type: 'VIF'
}
export type XoVm = BaseXoVm & {
id: Branded<'VM'>
type: 'VM'
vulnerabilities: { xsa468: boolean | { reason: string; driver?: string; version?: string } }
}
export type XoVmController = BaseXoVm & {
id: Branded<'VM-controller'>
type: 'VM-controller'
}
export type XoVmSnapshot = BaseXoVm & {
$snapshot_of: XoVm['id']
id: Branded<'VM-snapshot'>
snapshot_time: number
suspendVdi?: XoVdi['id']
type: 'VM-snapshot'
}
export type XoVmTemplate = BaseXoVm & {
id: Branded<'VM-template'>
isDefaultTemplate: boolean
template_info: {
arch?: string
disks: {
bootable: true
size: number
SR: string
type: string
}[]
install_methods: string[]
install_repository?: string
}
type: 'VM-template'
}
export type XoVtpm = BaseXapiXo & {
id: Branded<'VTPM'>
type: 'VTPM'
}
export type XapiXoRecord =
| XoAlarm
| XoGpuGroup
| XoHost
| XoMessage
| XoNetwork
| XoPbd
| XoPci
| XoPgpu
| XoPif
| XoPool
| XoSr
| XoVbd
| XoVdi
| XoVdiSnapshot
| XoVdiUnmanaged
| XoVgpu
| XoVgpuType
| XoVif
| XoVm
| XoVmController
| XoVmSnapshot
| XoVmTemplate
| XoVtpm
| XoSm
export type NonXapiXoRecord =
| AnyXoBackupArchive
| AnyXoJob
| AnyXoLog
| XoGroup
| XoProxy
| XoGroup
| XoProxy
| XoJob
| XoBackupRepository
| XoSchedule
| XoServer
| XoTask
| XoUser
export type XoRecord = XapiXoRecord | NonXapiXoRecord
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
export type AnyXoVdi = XoVdi | XoVdiSnapshot | XoVdiUnmanaged
export type AnyXoJob = XoJob | AnyXoBackupJob
export type AnyXoBackupJob = XoVmBackupJob | XoMetadataBackupJob | XoMirrorBackupJob
export type AnyXoLog = XoBackupLog | XoRestoreLog
export type AnyXoBackupArchive = XoVmBackupArchive | XoConfigBackupArchive | XoPoolBackupArchive

View File

@@ -1,13 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"declaration": true,
"strict": true,
"module": "ESNext",
"target": "ES2022",
"skipLibCheck": true,
"types": ["node"] // Only use types from Node.js and packages that are explicitly imported
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

View File

@@ -20,7 +20,7 @@ function main([inputPath = 0]) {
try {
data = JSON.parse(input)
} catch (error) {
// fall back to XML
// fallback to XML
log(parseXml(input))
return
}

View File

@@ -98,17 +98,6 @@ class AuditCore {
return record
}
async _importRecord(record) {
if (!record.id || !record.previousId) {
throw new Error('Invalid record')
}
await this._storage.put(record)
return {
id: record.id,
isValid: record.id === createHash(record),
}
}
async checkIntegrity(oldest, newest) {
const storage = this._storage

View File

@@ -7,7 +7,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.4.0",
"version": "0.3.1",
"engines": {
"node": ">=14"
},

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,21 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
},
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,49 +0,0 @@
{
"name": "@xen-orchestra/backup-archive",
"version": "1.0.0",
"description": "Helper to handle generic backup archives",
"license": "ISC",
"author": "",
"type": "module",
"main": "./dist/index.mjs",
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^6.0.1",
"@types/node": "^22.3.0",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0",
"fs-extra": "^11.1.0",
"sinon": "^18.0.0",
"tar-stream": "^3.1.8",
"@types/tar-stream": "^3.1.4",
"tmp": "^0.2.1",
"@vates/types": "^1.21.0"
},
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/fs": "^4.6.7",
"@xen-orchestra/backups": "^0.69.2",
"uuid": "^9.0.0",
"promise-toolbox": "^0.21.0",
"@vates/async-each": "^1.0.1",
"vhd-lib": "^4.14.7"
},
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test-integration": "tsc && node --test **/*.integ.mjs",
"clean": "rimraf dist/"
},
"private": true,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/backup-archive",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20.17"
}
}

View File

@@ -1,357 +0,0 @@
import RemoteHandlerAbstract from '@xen-orchestra/fs'
import { dirname, normalize } from '@xen-orchestra/fs/path'
import { MergeRemoteDisk, openDisk, openDiskChainFromPaths, RemoteDisk, isDiskFile } from '@xen-orchestra/backups/disks'
import { DEFAULT_MERGE_CONCURRENCY, DEFAULT_REMOVE_CONCURRENCY, ResolvedBackupCleanOptions } from './VmBackup.types.mjs'
import { asyncEach } from '@vates/async-each'
/**
* Tracks the disk chain for a single VDI across all backup snapshots.
* Owns merge and deletion decisions for its chain given which disks are still
* referenced by active backups (accumulated via addActiveDiskPaths).
*/
export class RemoteDiskLineage {
#handler: RemoteHandlerAbstract
#vdiDir: string
#opts: ResolvedBackupCleanOptions
// Concrete disk paths discovered in this VDI directory (normalized)
#diskPaths: Set<string> = new Set()
// Disk paths that exist on disk but failed to open (corrupted)
#brokenDiskPaths: Set<string> = new Set()
// child path: parent path (from disk header)
#parentOf: Map<string, string> = new Map()
// parent path: child path
#childOf: Map<string, string> = new Map()
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
#activeDiskPaths: Set<string> = new Set()
// Interrupted merges: normalized parent path { stateFilePath, chain }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
this.#handler = handler
this.#vdiDir = vdiDir
this.#opts = opts
}
#unregisterDisk(diskPath: string): void {
this.#diskPaths.delete(diskPath)
const parent = this.#parentOf.get(diskPath)
if (parent !== undefined) {
this.#parentOf.delete(diskPath)
this.#childOf.delete(parent)
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
this.#childOf.delete(diskPath)
this.#parentOf.delete(child)
}
}
/**
* Lists disk files and reads their headers to build the parent-child chain.
* Also detects interrupted merge state files.
*/
async init(): Promise<void> {
const files = await this.#handler.list(this.#vdiDir, { prependDir: true })
for (const filePath of files) {
if (isDiskFile(filePath)) {
this.#diskPaths.add(normalize(filePath))
}
}
const uuidToPath = new Map<string, string>()
for (const diskPath of this.#diskPaths) {
let disk: RemoteDisk | undefined
try {
disk = await openDisk({ handler: this.#handler as any, path: diskPath })
} catch (error: any) {
if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
throw error
}
this.#opts.logWarn('failed to open disk', { path: diskPath, error })
this.#brokenDiskPaths.add(diskPath)
continue
}
try {
const uuid = disk.getUuid()
// Detect Disks with the same UUIDs
// Due to a bug introduced in a1bcd35e2
const existingPath = uuidToPath.get(uuid)
if (existingPath !== undefined) {
this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
let existingDisk: RemoteDisk | undefined
try {
existingDisk = await openDisk({ handler: this.#handler as any, path: existingPath })
if (existingDisk.containsAllDataOf(disk)) {
this.#opts.logWarn('dropping duplicate disk, existing is superset', { dropped: diskPath })
this.#unregisterDisk(diskPath)
continue
} else if (disk.containsAllDataOf(existingDisk)) {
this.#opts.logWarn('dropping duplicate disk, new is superset', { dropped: existingPath })
this.#unregisterDisk(existingPath)
} else {
this.#opts.logWarn('duplicate disks have different content, keeping first-seen', { dropped: diskPath })
this.#unregisterDisk(diskPath)
continue
}
} catch (error) {
this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
this.#unregisterDisk(existingPath)
} finally {
await existingDisk?.close()
}
}
uuidToPath.set(uuid, diskPath)
if (disk.isDifferencing()) {
const parentPath = disk.getParentPath()
this.#parentOf.set(diskPath, parentPath)
if (this.#childOf.has(parentPath)) {
const error = new Error('this script does not support multiple children', {
cause: {
parent: parentPath,
child1: this.#childOf.get(parentPath),
child2: diskPath,
},
})
throw error
}
this.#childOf.set(parentPath, diskPath)
}
} catch (error) {
this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
} finally {
await disk.close()
}
}
this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
this.#handler as any,
this.#vdiDir,
files,
uuidToPath,
this.#childOf
)
}
/**
* Returns true if the disk path exists on disk and opened without errors during init.
* Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
*/
isPathAccessible(diskPath: string): boolean {
return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
}
/** Returns true if the disk exists but failed to open during init. */
isBroken(diskPath: string): boolean {
return this.#brokenDiskPaths.has(diskPath)
}
/** Verifies that every parent referenced in disk headers exists in the directory. */
async check(): Promise<void> {
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent)) {
this.#opts.logWarn('parent disk is missing', { child, parent })
}
}
}
/**
* Declares disk paths as active for this lineage.
* Called by each archive that references this lineage after it determines it is complete.
* Accumulated across all referencing archives before clean() is called.
*/
addActiveDiskPaths(diskPaths: string[]): void {
for (const path of diskPaths) {
this.#activeDiskPaths.add(normalize(path))
}
}
/**
* For each orphan disk:
* - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
* - If it has no active descendant → delete it (if opts.remove).
*
* Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
* Deletes orphan merge state files (for missing disks) when opts.remove is true.
*
* Requires addActiveDiskPaths() to have been called by all referencing archives before this.
* @returns Set of deleted disk paths.
*/
async clean({
remove = this.#opts.remove ?? false,
merge = this.#opts.merge ?? false,
}: { remove?: boolean; merge?: boolean } = {}): Promise<{ deleted: Set<string>; mergedSizes: Map<string, number> }> {
const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
const toDelete = new Set<string>()
const toMerge: { chain: string[]; isResuming: boolean }[] = []
const visited = new Set<string>()
// Walk from an orphan toward its descendants.
// Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
// or undefined if the subtree is entirely orphaned (mark for deletion).
const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
if (visited.has(diskPath)) {
return undefined
}
visited.add(diskPath)
if (!orphanDisks.has(diskPath)) {
// Active disk, anchor of the merge chain
return [diskPath]
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
const chain = getUsedChildChainOrDelete(child)
if (chain !== undefined) {
chain.unshift(diskPath)
return chain
}
}
toDelete.add(diskPath)
return undefined
}
// Process interrupted merges first so their disks are protected from the orphan loop
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
if (!this.#diskPaths.has(parentPath)) {
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
if (remove) {
this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
let chain: string[] | undefined
if (stateChain !== undefined) {
const existing = stateChain.filter(p => this.#diskPaths.has(p))
if (existing.length >= 2) chain = existing
}
if (chain !== undefined) {
chain.forEach(p => visited.add(p))
toMerge.push({ chain, isResuming: true })
}
}
for (const orphan of orphanDisks) {
if (!visited.has(orphan)) {
const parent = this.#parentOf.get(orphan)
if (parent === undefined || !orphanDisks.has(parent)) {
const chain = getUsedChildChainOrDelete(orphan)
if (chain !== undefined && chain.length > 1) {
toMerge.push({ chain, isResuming: false })
}
}
}
}
if (remove) {
await asyncEach(
Array.from(toDelete),
async path => {
this.#opts.logInfo('deleting unused disk', { path })
try {
const disk = await openDisk({ handler: this.#handler as any, path, ignoreBlockIndexes: true })
await disk.unlink({ force: true })
} catch (error) {
this.#opts.logWarn('failed to delete unused disk', { path, error })
}
},
{ concurrency: DEFAULT_REMOVE_CONCURRENCY }
)
}
if (toMerge.length > 0) {
this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
}
// mergeTargetPath: final size of the disk everything was merged into
const mergedSizes = new Map<string, number>()
if (merge) {
await asyncEach(
toMerge,
async ({ chain, isResuming }) => {
const { finalDiskSize, mergeTargetPath } = await this.#mergeChain(chain, isResuming)
mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
},
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
)
}
await this.#cleanOrphanDataFiles(remove)
return { deleted: toDelete, mergedSizes }
}
/**
* Merges ancestors into the active child at the end of the chain.
* Returns the final size of the merge target and its path.
*/
async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
const parentPath = chain[0]
// The last disk in the chain is the active one that everything gets merged into
const mergeTargetPath = chain[chain.length - 1]
this.#opts.logInfo('merging disk chain', { chain })
const merger = new MergeRemoteDisk(this.#handler as any, {
logInfo: this.#opts.logInfo,
removeUnused: this.#opts.remove,
mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
onProgress: this.#opts.onProgress,
})
const parentDisk = await openDisk({ handler: this.#handler as any, path: parentPath, force: isResuming })
const childDisk = await openDiskChainFromPaths({
handler: this.#handler as any,
paths: chain.slice(1),
force: isResuming,
})
const { finalDiskSize } = await merger.merge(parentDisk, childDisk)
return { finalDiskSize, mergeTargetPath }
}
/**
* Validates each disk integrity, then scans discovered data subdirs
* and deletes any files not associated to a known disk.
*/
async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
const claimedFiles = new Set<string>()
for (const diskPath of this.#diskPaths) {
let claimed: string[] = []
const disk = await openDisk({ handler: this.#handler as any, path: diskPath, ignoreBlockIndexes: true })
try {
await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
claimed = await disk.listAssociatedFiles(this.#vdiDir)
} finally {
await disk.close()
}
for (const f of claimed) claimedFiles.add(normalize(f))
}
const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
for (const dataDir of dataDirs) {
const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
for (const item of items) {
if (!claimedFiles.has(normalize(item))) {
this.#opts.logWarn('orphaned data file', { path: item })
if (remove) {
await this.#handler.unlink(item).catch(async (err: any) => {
if (err?.code === 'EISDIR') {
await this.#handler.rmtree(item)
} else {
this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
}
})
}
}
}
}
}
}

View File

@@ -1,66 +0,0 @@
import RemoteHandlerAbstract from '@xen-orchestra/fs'
export interface PartialBackupMetadata {
mode: 'full' | 'delta'
xva?: string
vhds?: Record<string, string>
vdis?: Record<string, unknown>
isVhdDifferencing?: Record<string, boolean>
size?: number
jobId: string
scheduleId: string
timestamp: number
}
export const DEFAULT_MERGE_CONCURRENCY = 1
export const DEFAULT_REMOVE_CONCURRENCY = 4
export interface BackupCleanOptions {
fix?: boolean
merge?: boolean
remove?: boolean
mergeBlockConcurrency?: number
onProgress?: (progress: { total: number; done: number }) => void
logInfo?: (message: any, opts?: object) => void
logWarn?: (message: any, opts?: object) => void
}
export type ResolvedBackupCleanOptions = BackupCleanOptions & {
logInfo: (message: any, opts?: object) => void
logWarn: (message: any, opts?: object) => void
}
export interface ArchiveCleanOptions {
remove?: boolean
merge?: boolean
mergedSizes?: Map<string, number>
}
export interface CheckResult {
isValid: boolean
// Files referenced by metadata but absent on disk
missingDisks?: string[]
// Disk files that exist but failed to open (corrupted)
brokenDisks?: string[]
// Files present on disk but not referenced by metadata
orphans?: string[]
// Files referenced by at least one valid backup
linked?: string[]
}
export type CleanResult = {
removedFiles: string[]
merge: boolean
}
export interface VmBackupInterface {
handler: RemoteHandlerAbstract
// metadataPath: string
// metadata: PartialBackupMetadata
rootPath: string
opts: BackupCleanOptions
init(): Promise<void>
check(): Promise<CheckResult>
clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
getAssociatedFiles(opts: object): Array<string>
}

View File

@@ -1,260 +0,0 @@
import RemoteHandlerAbstract from '@xen-orchestra/fs'
import { basename, normalize } from '@xen-orchestra/fs/path'
import { resolve } from 'node:path'
import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
import {
ArchiveCleanOptions,
BackupCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
DEFAULT_MERGE_CONCURRENCY,
} from './VmBackup.types.mjs'
import { cleanOrphanDiskDirs } from '@xen-orchestra/backups/disks'
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
export class VmBackupDirectory implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
files: Array<string> = new Array()
orphans: Set<string> = new Set()
backupArchives: Map<string, VmBackupInterface> = new Map()
opts: ResolvedBackupCleanOptions
// Cached result of the last check() call; invalidated by init()
#checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
#uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
#remoteAdapter: RemoteAdapter
constructor(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions = {
fix: true,
merge: false,
remove: false,
logInfo,
logWarn,
}
) {
this.handler = handler
this.rootPath = vmBackupPath
this.opts = {
...opts,
fix: opts.fix ?? true,
merge: opts.merge ?? false,
remove: opts.remove ?? false,
logInfo: opts.logInfo ?? logInfo,
logWarn: opts.logWarn ?? logWarn,
}
this.#remoteAdapter = new RemoteAdapter(handler)
}
async init() {
this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
this.#checkResult = undefined
this.#uniqueLineages = new Map()
for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
let metadata: PartialBackupMetadata | undefined = undefined
try {
metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
} catch (error) {
this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
}
if (metadata !== undefined) {
try {
const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
this.backupArchives.set(fullPath, backupArchive)
} catch (error: any) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
}
}
}
}
getAssociatedFiles({ prefix = false }) {
const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
return prefix ? files : files.map(file => basename(file))
}
async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
await this.#checkCacheCount()
for (const backupArchive of this.backupArchives.values()) {
await backupArchive.check()
}
// allUsedFiles is used for root-level orphan detection only.
// Active disk paths are accumulated per-lineage by each archive during check() above.
const allUsedFiles = new Set<string>([
...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
...this.getAssociatedFiles({ prefix: true }),
])
for (const lineage of this.#uniqueLineages!.values()) {
await lineage.check()
}
const orphans = this.files.filter(file => !allUsedFiles.has(file))
const linked = Array.from(allUsedFiles)
this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
return this.#checkResult
}
async clean({
remove = this.opts.remove ?? false,
merge = this.opts.merge ?? false,
}: ArchiveCleanOptions = {}): Promise<CleanResult> {
// Use cached check result if available, otherwise run check now
const { orphans } = this.#checkResult ?? (await this.check())
let cacheNeedsRegen = false
// Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
const allMergedSizes = new Map<string, number>()
await asyncEach(
Array.from(this.#uniqueLineages!.entries()),
async ([_vdiDir, lineage]) => {
const { mergedSizes, deleted } = await lineage.clean({ remove, merge })
if (deleted.size > 0) {
cacheNeedsRegen = true
}
for (const [diskPath, size] of mergedSizes) {
allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
}
},
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
)
// Delete VDI directories not referenced by any archive
const coveredDirs = new Set(this.#uniqueLineages!.keys())
await cleanOrphanDiskDirs(this.handler as any, this.rootPath, coveredDirs, {
remove,
logWarn: this.opts.logWarn,
logInfo: this.opts.logInfo,
})
// Let each archive clean its own files (e.g. remove metadata for incomplete backups)
// and update metadata with merged sizes if applicable
await asyncEach(
Array.from(this.backupArchives.values()),
async (archive: VmBackupInterface) => {
const { removedFiles, merge: didMerge } = await archive.clean({ remove, mergedSizes: allMergedSizes })
if (removedFiles.length > 0 || didMerge) {
cacheNeedsRegen = true
}
},
{ concurrency: 2 }
)
if (cacheNeedsRegen) {
await this.#regenerateCache()
}
// Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
if (remove) {
await asyncEach(
orphans,
async (orphan: string) => {
try {
await this.handler.unlink(orphan)
} catch (error: any) {
if (error?.code === 'EISDIR') {
this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
} else if (error?.code !== 'ENOENT') {
throw error
}
}
},
{ concurrency: 2 }
)
}
return { removedFiles: orphans, merge: allMergedSizes.size > 0 }
}
async #checkCacheCount(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const existingCache = await this.#remoteAdapter._readCache(cachePath)
const actual = existingCache === undefined ? undefined : Object.keys(existingCache).length
const expected = this.backupArchives.size
if (actual !== undefined && actual !== expected) {
this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
}
}
async #regenerateCache(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const cache: Record<string, object> = {}
for (const [path, archive] of this.backupArchives.entries()) {
const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
cache[path] = { _filename: path, id: path, ...a.metadata }
}
await this.#remoteAdapter._writeCache(cachePath, cache)
}
async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
let backupArchive: VmBackupInterface
try {
if (metadata.mode === 'full') {
backupArchive = new VmFullBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
metadata.xva!,
this.opts
)
} else if (metadata.mode === 'delta') {
const rawDiskPaths = metadata.vhds
const diskPaths =
rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
backupArchive = new VmIncrementalBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
diskPaths,
this.opts,
this.#uniqueLineages
)
} else {
throw new Error(`Mode ${metadata.mode} not supported`)
}
} catch (error) {
this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
}
await backupArchive.init()
return backupArchive
}
/**
* Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
* The `lock` option is accepted but ignored (locking is the caller's responsibility).
*/
static async cleanVm(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions & { lock?: boolean } = {}
) {
const { lock: _lock, ...cleanOpts } = opts
const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
await dir.init()
await dir.check()
const { merge } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
return { merge }
}
}

View File

@@ -1,144 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
import { rimraf } from 'rimraf'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import tar from 'tar-stream'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, vmBackupDir
const vmUuid = 'test-vm-uuid'
const rootPath = `xo-vm-backups/${vmUuid}`
async function createMinimalXva() {
const pack = tar.pack()
pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
pack.finalize()
const chunks = []
for await (const chunk of pack) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
await fs.mkdirp(`${tempDir}/${rootPath}`)
vmBackupDir = new VmBackupDirectory(handler, rootPath)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
async function createFullBackupMetadata(name, xvaName) {
const metadata = {
mode: 'full',
xva: `${rootPath}/${xvaName}`,
vm: { uuid: vmUuid },
jobId: uniqueId(),
timestamp: Date.now(),
}
await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
return metadata
}
describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
test('init() loads full backup archives from metadata files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await createFullBackupMetadata('backup2.json', 'backup2.xva')
await vmBackupDir.init()
assert.equal(vmBackupDir.backupArchives.size, 2)
})
test('getAssociatedFiles() returns cache files', async () => {
// Create a cache file
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
await vmBackupDir.init()
const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
assert.equal(filesWithPrefix.length, 1)
assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
})
test('check() identifies orphan files', async () => {
// Create a valid backup
await createFullBackupMetadata('backup1.json', 'backup1.xva')
// Create orphan files (not referenced by any metadata)
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
await vmBackupDir.init()
const { orphans, linked } = await vmBackupDir.check()
assert.ok(orphans.some(f => f.includes('orphan.xva')))
assert.ok(orphans.some(f => f.includes('random-file.txt')))
assert.ok(linked.some(f => f.includes('backup1.json')))
assert.ok(linked.some(f => f.includes('backup1.xva')))
})
test('clean() removes orphan files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.json'))
assert.ok(remainingFiles.includes('backup1.xva'))
assert.ok(!remainingFiles.includes('orphan.xva'))
})
test('clean() preserves cache.json.gz', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
await vmBackupDir.init()
await vmBackupDir.clean()
const remainingFiles = await handler.list(rootPath)
assert.equal(remainingFiles.length, 3)
assert.ok(remainingFiles.includes('cache.json.gz'))
})
test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
})
test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept alongside its valid backup')
})
})

View File

@@ -1,139 +0,0 @@
import { basename, normalize } from '@xen-orchestra/fs/path'
import assert from 'node:assert'
import { FileDescriptor } from '@xen-orchestra/fs'
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import RemoteHandlerAbstract from '@xen-orchestra/fs'
const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
// https://tools.ietf.org/html/rfc1952.html#page-5
Buffer.from('1F8B', 'hex'),
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
Buffer.from('28B52FFD', 'hex'),
]
const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
return true
}
}
return false
}
const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
if (size <= 1024 || size % 512 !== 0) {
return false
}
const buf = Buffer.allocUnsafe(1024)
assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
return buf.every(byte => byte === 0)
}
export async function isValidXva(handler: RemoteHandlerAbstract, filePath: string): Promise<boolean> {
// size is longer when encrypted + reading part of an encrypted file is not implemented
if (handler.isEncrypted) {
return true
}
try {
const fd = await handler.openFile(filePath, 'r')
try {
const size = await handler.getSize(filePath)
if (size < 20) {
// neither a valid gzip nor tar
return false
}
return (await isCompressedFile(handler, fd))
? true // compressed files cannot be validated at this time
: await isValidTar(handler, size, fd)
} finally {
handler.closeFile(fd).catch(noop)
}
} catch (error) {
// never throw, log and report as valid to avoid side effects
return true
}
}
const noop = (): void => {}
export class VmFullBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
xvaPath: string
isValid?: boolean
metadataPath: string
metadata: PartialBackupMetadata
opts: ResolvedBackupCleanOptions
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
opts: ResolvedBackupCleanOptions
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.xvaPath = normalize(xvaPath)
this.opts = opts
}
async init() {}
async check(): Promise<CheckResult> {
if (this.isValid === undefined) {
this.isValid = await isValidXva(this.handler, this.xvaPath)
}
if (!this.isValid) {
this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
}
return { isValid: this.isValid! }
}
async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (this.isValid === undefined) {
await this.check()
}
const removedFiles: string[] = []
if (!this.isValid) {
removedFiles.push(this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`)
}
if (remove) {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`)
}
}
}
return { removedFiles, merge: false }
}
getAssociatedFiles({ prefix = false }): Array<string> {
let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
if (!this.isValid) {
validFiles = []
}
return prefix ? validFiles : validFiles.map(file => basename(file))
}
}

View File

@@ -1,687 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
import { dirname, basename } from 'node:path'
import { rimraf } from 'rimraf'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
const rootPath = 'xo-vm-backups/VMUUID/'
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
jobId = uniqueId()
vdiId = uniqueId()
relativePath = `vdis/${jobId}/${vdiId}`
basePath = `${rootPath}/${relativePath}`
vdiId2 = uniqueId()
relativePath2 = `vdis/${jobId}/${vdiId2}`
basePath2 = `${rootPath}/${relativePath2}`
await fs.mkdirp(`${tempDir}/${basePath}`)
await fs.mkdirp(`${tempDir}/${basePath2}`)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
async function generateVhd(path, opts = {}) {
let vhd
let dataPath = path
if (opts.useAlias) {
await handler.mkdir(dirname(path) + '/data/')
dataPath = dirname(path) + '/data/' + basename(path)
}
if (opts.mode === 'directory') {
await handler.mkdir(dataPath)
vhd = new VhdDirectory(handler, dataPath)
} else {
const fd = await handler.openFile(dataPath, 'wx')
vhd = new VhdFile(handler, fd)
}
vhd.header = { ...VHDHEADER, ...opts.header }
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
if (vhd.header.parentUuid) {
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
if (opts.useAlias === true) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
if (opts.blocks) {
for (const blockId of opts.blocks) {
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
}
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
return vhd
}
test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
// metadata references a non-existent disk so the lineage is created for this vdiDir
// and notReallyAVhd.vhd is treated as an unreferenced orphan
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
assert.ok(logged.includes('failed to open disk'))
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent, should be deleted
await generateVhd(`${basePath}/abandoned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// one orphan, which is a full vhd, no parent : should stay
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan in the metadata : should stay
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
})
)
// clean
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
// abandoned has a missing parent, orphan and child have no active backup reference
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
assert.equal(detectedMissingParent.length, 1)
// we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
})
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.json is not here
],
})
)
await generateVhd(`${basePath}/abandoned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
let matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.vhd is not here anymore
],
}),
{ flags: 'w' }
)
logged = ''
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after metadata deletion'
)
})
test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
// A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
// Metadata references the corrupt VHD as if it were a valid active disk
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/broken.vhd`],
})
)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
logInfo: () => {},
logWarn: logInfo,
})
assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
// metadata removed because backup is incomplete
assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
// broken VHD removed via the orphan path in lineage.clean()
assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
})
test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
// Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
// Each snapshot is the active disk for exactly one backup metadata file.
//
// Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
// own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
// RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
// their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
// deleted — destroying two live backups.
const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
})
await generateVhd(`${basePath}/snapshot3.vhd`, {
header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
})
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata3.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
assert.equal(remaining.length, 3, 'no extra files should remain')
// All three metadata files must also survive (all backups are complete)
const rootFiles = await handler.list(rootPath)
assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
})
test('it handles multiple VDI directories independently', async () => {
// vdiDir1: diskA1 (base) <- diskA2, plus an orphan
const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
await generateVhd(`${basePath}/diskA2.vhd`, {
header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
})
await generateVhd(`${basePath}/orphanA.vhd`)
// vdiDir2: diskB1 (base) <- diskB2, plus an orphan
const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
await generateVhd(`${basePath2}/diskB2.vhd`, {
header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
})
await generateVhd(`${basePath2}/orphanB.vhd`)
// archive1 covers both base disks, archive2 covers both child disks
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining1 = await handler.list(basePath)
assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
const remaining2 = await handler.list(basePath2)
assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
})
test('it merges delta of non destroyed chain', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000, // a size too small
vhds: [
`${relativePath}/grandchild.vhd`, // grand child should not be merged
`${relativePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
// a grand child
await generateVhd(`${basePath}/grandchild.vhd`, {
header: {
parentUnicodeName: 'child.vhd',
parentUuid: child.footer.uuid,
},
})
let logged = []
const logInfo = message => {
logged.push(message)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
assert.equal(logged[0], `Disk chain needs merging`)
logged = []
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
assert.equal(logged.includes(`merging disk chain`), true)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 2)
assert.equal(remainingVhds.includes('child.vhd'), true)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
assert.equal(metadata.size, 104448, 'metadata.size should be updated after merge')
})
test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
// ancestor -> child -> grandchild
// only grandchild is referenced in metadata: both ancestor and child are orphans
// they must be merged as a single 3-disk chain, not processed separately
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000,
vhds: [`${relativePath}/grandchild.vhd`],
})
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
const child = await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
})
await generateVhd(`${basePath}/grandchild.vhd`, {
header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
})
const mergeChains = []
const logInfo = (message, opts) => {
if (message === 'merging disk chain') mergeChains.push(opts.chain)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
// all three disks must be handled in exactly one merge, not two separate operations
assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
assert.equal(remainingVhds.includes('child.vhd'), false)
})
test('it resumes an interrupted merge with chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [5, 6],
})
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
blocks: [7, 8],
})
await handler.writeFile(
`${basePath}/.ancestor.vhd.merge.json`,
JSON.stringify({
parent: { uuid: '0' },
child: { uuid: '0' },
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
currentBlock: 3,
mergedDataSize: 3 * 2 * 1024 * 1024,
step: 'mergeBlocks',
diskSize: 0,
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const files = await handler.list(basePath)
assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
})
test('it resumes an interrupted merge without chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 209920,
vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
// a merge in progress file
await handler.writeFile(
`${basePath}/.orphan.vhd.merge.json`,
JSON.stringify({
parent: {
header: orphan.header.checksum,
},
child: {
header: child.header.checksum,
},
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 1)
assert.equal(remainingVhds.includes('child.vhd'), true)
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
describe('tests multiple combination ', { concurrency: 1 }, () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
// a broken VHD
if (useAlias) {
await handler.mkdir(basePath + '/data')
}
const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
if (vhdMode === 'directory') {
await handler.mkdir(brokenVhdDataPath)
} else {
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
}
if (useAlias) {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
await generateVhd(`${basePath}/abandoned.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// an ancestor of a vhd present in metadata
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
blocks: [1, 3],
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: ancestor.footer.uuid,
},
blocks: [1, 2],
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: child.footer.uuid,
},
blocks: [2, 3],
})
// an older parent that was merging in clean
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
useAlias,
mode: vhdMode,
})
// a clean vhd in metadata
const clean = await generateVhd(`${basePath}/clean.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: cleanAncestor.footer.uuid,
},
})
await handler.writeFile(
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
JSON.stringify({
parent: {
header: cleanAncestor.header.checksum,
},
child: {
header: clean.header.checksum,
},
})
)
// the metadata file
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
`${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
`${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
],
})
)
if (!useAlias && vhdMode === 'directory') {
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
} catch (err) {
assert.strictEqual(
err.code,
'NOT_SUPPORTED',
'Merging directory without alias should raise a not supported error'
)
return
}
assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 4404224)
// cache.json.gz must be regenerated when metadata changes after a merge
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after a merge'
)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
if (useAlias) {
const dataSurvivors = await handler.list(basePath + '/data')
// the goal of the alias : do not move a full folder
assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
assert.equal(survivors.length, 4) // the 3 ok + data
assert.equal(dataSurvivors.length, 3)
} else {
assert.equal(survivors.includes('clean.vhd'), true)
assert.equal(survivors.includes('child.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd'), true)
assert.equal(survivors.length, 3)
}
})
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
// metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('check all types of aliases, corrupted, missing or not', async () => {
// valid alias pointing to an existing data file
await handler.mkdir(`${basePath}/data`)
await generateVhd(`${basePath}/data/ok.vhd`)
await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
// broken alias pointing to a missing data file
await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
// alias pointing to an existing but corrupt data file
await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
// data file without alias
await generateVhd(`${basePath}/data/missingalias.vhd`)
// only the valid alias is referenced by a complete backup
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
// only ok.alias.vhd has survived
const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
assert.equal(aliases.length, 1)
assert.equal(aliases[0], 'ok.alias.vhd')
// corrupt.vhd and missingalias.vhd must be gone
const data = await handler.list(`${basePath}/data`)
assert.equal(data.length, 1)
assert.equal(data[0], 'ok.vhd')
})

View File

@@ -1,174 +0,0 @@
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
ResolvedBackupCleanOptions,
VmBackupInterface,
PartialBackupMetadata,
} from './VmBackup.types.mjs'
import RemoteHandlerAbstract from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
export class VmIncrementalBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
metadataPath: string
metadata: PartialBackupMetadata
readonly diskPaths: Array<string>
rootPath: string
opts: ResolvedBackupCleanOptions
diskLineages: Map<string, RemoteDiskLineage> = new Map()
#isChecked = false
#isComplete = false
get isChecked(): boolean {
return this.#isChecked
}
get isComplete(): boolean {
return this.#isComplete
}
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
diskPaths: Array<string>, // xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/<snapshotUuid>.alias.vhd (one per VDI)
opts: ResolvedBackupCleanOptions,
lineageRegistry?: Map<string, RemoteDiskLineage>
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.diskPaths = diskPaths.map(path => normalize(path))
this.opts = opts
if (lineageRegistry !== undefined) {
this.diskLineages = lineageRegistry
}
}
async init(): Promise<void> {
// Build one RemoteDiskLineage per VDI directory derived from diskPaths
try {
for (const diskPath of this.diskPaths) {
const vdiDir = dirname(diskPath)
if (!this.diskLineages.has(vdiDir)) {
const lineage = new RemoteDiskLineage(this.handler, vdiDir, this.opts)
await lineage.init()
this.diskLineages.set(vdiDir, lineage)
}
}
} catch (error: any) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn('failed to scan VDI directories', { error })
}
}
async check(): Promise<CheckResult> {
const missingDisks: string[] = []
const brokenDisks: string[] = []
if (this.diskPaths.length === 0) {
this.opts.logWarn('incremental backup has no disk paths', { metadataPath: this.metadataPath })
this.#isComplete = false
} else {
for (const diskPath of this.diskPaths) {
const lineage = this.diskLineages.get(dirname(diskPath))
if (lineage === undefined || !lineage.isPathAccessible(diskPath)) {
if (lineage?.isBroken(diskPath)) {
this.opts.logWarn('disk check error', { path: diskPath })
brokenDisks.push(diskPath)
} else {
this.opts.logWarn('disk file missing', { path: diskPath })
missingDisks.push(diskPath)
}
}
}
this.#isComplete = missingDisks.length === 0 && brokenDisks.length === 0
if (!this.#isComplete) {
this.opts.logWarn('incremental backup is incomplete', {
metadataPath: this.metadataPath,
missingDisks,
brokenDisks,
})
}
}
if (this.#isComplete) {
for (const lineage of this.diskLineages.values()) {
lineage.addActiveDiskPaths(this.diskPaths)
}
}
this.#isChecked = true
return { isValid: this.#isComplete, missingDisks, brokenDisks }
}
/**
* Removes the metadata file if the backup is incomplete (missing disks).
* If mergedSizes is provided, updates metadata with the total merged size for this archive's disks.
* Actual disk merge/deletion is handled by RemoteDiskLineage.
*/
async clean({ remove = this.opts.remove ?? false, mergedSizes }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (!this.#isChecked) {
await this.check()
}
const removedFiles: string[] = []
if (!this.#isComplete) {
removedFiles.push(this.metadataPath)
}
if (remove) {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
let mergedSize = 0
if (mergedSizes !== undefined) {
for (const diskPath of this.diskPaths) {
mergedSize += mergedSizes.get(diskPath) ?? 0
}
if (mergedSize > 0) {
await this.updateMetadata(mergedSize)
}
}
return { removedFiles, merge: mergedSize > 0 }
}
/**
* Updates metadata.size after a merge and writes the updated metadata file to disk.
* Also resets isVhdDifferencing so all VDIs are marked as full (non-differencing).
*/
async updateMetadata(mergedSize: number): Promise<void> {
this.metadata.size = mergedSize
if (this.metadata.vdis !== undefined) {
this.metadata.isVhdDifferencing = {}
for (const id of Object.keys(this.metadata.vdis)) {
this.metadata.isVhdDifferencing[id] = false
}
}
await this.handler.writeFile(this.metadataPath, JSON.stringify(this.metadata), { flags: 'w' })
}
/**
* Returns the metadata file path and active disk paths if the backup is complete, empty otherwise.
* Must be called after check().
*/
getAssociatedFiles({ prefix = false }): Array<string> {
if (!this.#isComplete) {
return []
}
const files = [this.metadataPath, ...this.diskPaths]
return prefix ? files : files.map(file => basename(file))
}
}

View File

@@ -1,92 +0,0 @@
// a valid footer of a 2
export const VHDFOOTER = {
cookie: 'conectix',
features: 2,
fileFormatVersion: 65536,
dataOffset: 512,
timestamp: 0,
creatorApplication: 'caml',
creatorVersion: 1,
creatorHostOs: 0,
originalSize: 53687091200,
currentSize: 53687091200,
diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
diskType: 3,
checksum: 4294962945,
uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
saved: '',
hidden: '',
reserved: '',
}
export const VHDHEADER = {
cookie: 'cxsparse',
dataOffset: undefined,
tableOffset: 2048,
headerVersion: 65536,
maxTableEntries: 25600,
blockSize: 2097152,
checksum: 4294964241,
parentUuid: null,
parentTimestamp: 0,
reserved1: 0,
parentUnicodeName: '',
parentLocatorEntry: [
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
],
reserved2: '',
}

View File

@@ -1,16 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"declaration": true,
"strict": true,
"module": "nodenext",
"moduleResolution": "nodenext",
"target": "ES2022",
"rootDir": "./src",
"skipLibCheck": true,
"noImplicitAny": false,
"types": ["node"] // Only use types from Node.js and packages that are explicitly imported
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

View File

@@ -4,10 +4,6 @@ Usage:
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
Detects and repair issues with VM backups.
Options:

View File

@@ -20,10 +20,6 @@ Usage:
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
Detects and repair issues with VM backups.
Options:

View File

@@ -1,81 +1,13 @@
import { asyncEach } from '@vates/async-each'
import { asyncMap } from '@xen-orchestra/async-map'
import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
import { getSyncedHandler } from '@xen-orchestra/fs'
import getopts from 'getopts'
import { basename, dirname, join } from 'path'
import { basename, dirname } from 'path'
import Disposable from 'promise-toolbox/Disposable'
import { pathToFileURL } from 'url'
async function cleanFiles(vmDirs, { fix, merge, remove }) {
let nbSucceed = 0
let nbFailed = 0
let nbStarted = 0
const total = vmDirs.length
// don't assume all the files uses the same remote
await asyncEach(
vmDirs,
vmDir =>
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
try {
nbStarted++
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
nbSucceed++
} catch (error) {
nbFailed++
console.error('adapter.cleanVm', vmDir, error, { nbStarted, nbSucceed, nbFailed, total })
}
}),
{ concurrency: 2 }
)
console.log('done', { nbSucceed, nbFailed })
}
async function cleanRemote(url, { fix, merge, remove }) {
let nbSucceed = 0
let nbFailed = 0
let nbStarted = 0
await Disposable.use(getSyncedHandler({ url }), async handler => {
const remoteAdapter = new RemoteAdapter(handler)
const vmUuids = await remoteAdapter.listAllVms()
const total = vmUuids.length
await asyncEach(
vmUuids,
async vmUuid => {
try {
nbStarted++
await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
nbSucceed++
} catch (error) {
nbFailed++
console.error('adapter.cleanVm', vmUuid, error, { nbStarted, nbSucceed, nbFailed, total })
}
},
{ concurrency: 2 }
)
})
console.log('done', { nbSucceed, nbFailed })
}
export default async function cleanVms(args) {
const {
_: vmDirs,
fix,
remote,
remove,
merge,
} = getopts(args, {
const { _, fix, remove, merge } = getopts(args, {
alias: {
fix: 'f',
remove: 'r',
@@ -88,9 +20,19 @@ export default async function cleanVms(args) {
},
})
if (remote) {
return cleanRemote(remote, { fix, merge, remove })
} else {
return cleanFiles(vmDirs, { fix, merge, remove })
}
await asyncMap(_, vmDir =>
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
try {
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
} catch (error) {
console.error('adapter.cleanVm', vmDir, error)
}
})
)
}

View File

@@ -6,10 +6,9 @@
"preferGlobal": true,
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@vates/async-each": "^1.0.1",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.69.4",
"@xen-orchestra/fs": "^4.6.7",
"@xen-orchestra/backups": "^0.57.0",
"@xen-orchestra/fs": "^4.3.0",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",
"lodash": "^4.17.15",
@@ -28,7 +27,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "1.1.8",
"version": "1.0.28",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -4,14 +4,12 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { importIncrementalVm } from './_incrementalVm.mjs'
import { Task } from './Task.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { VhdNegative, VhdSynthetic } from 'vhd-lib'
import { decorateClass } from '@vates/decorate-with'
import { createLogger } from '@xen-orchestra/log'
import { dirname, join } from 'node:path'
import pickBy from 'lodash/pickBy.js'
import { defer } from 'golike-defer'
import { NegativeDisk } from '@xen-orchestra/disk-transform'
import { openDiskChain } from './disks/openDiskChain.mjs'
import { resetVmOtherConfig } from './_otherConfig.mjs'
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
async function resolveUuid(xapi, cache, uuid, type) {
@@ -30,10 +28,10 @@ export class ImportVmBackup {
metadata,
srUuid,
xapi,
settings: { additionalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
}) {
this._adapter = adapter
this._importIncrementalVmSettings = { additionalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
this._metadata = metadata
this._srUuid = srUuid
this._xapi = xapi
@@ -61,12 +59,12 @@ export class ImportVmBackup {
const metadata = this._metadata
const { mapVdisSrs } = this._importIncrementalVmSettings
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
const disks = {}
const metadataDir = dirname(metadata._filename)
const streams = {}
const metdataDir = dirname(metadata._filename)
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
for (const [vdiRef, vdi] of Object.entries(vdis)) {
const vhdPath = join(metadataDir, vhds[vdiRef])
const vhdPath = join(metdataDir, vhds[vdiRef])
let xapiDisk
try {
@@ -78,7 +76,7 @@ export class ImportVmBackup {
let snapshotCandidate, backupCandidate
if (xapiDisk !== undefined) {
debug('found disks, will search its snapshots', { snapshots: xapiDisk.snapshots })
debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
for (const snapshotRef of xapiDisk.snapshots) {
const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
@@ -97,7 +95,7 @@ export class ImportVmBackup {
// have a corresponding backup more recent than metadata ?
const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
if (pathToSnapshotData === undefined) {
debug('no backup linked to this snapshot')
debug('no backup linked to this snaphot')
continue
}
if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
@@ -112,21 +110,20 @@ export class ImportVmBackup {
}
}
let disk
const backupWithSnapshotPath = join(metadataDir, backupCandidate ?? '')
let stream
const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
if (vhdPath === backupWithSnapshotPath) {
// all the data are already on the host
debug('direct reuse of a snapshot')
disk = null
stream = null
vdis[vdiRef].baseVdi = snapshotCandidate
// go next disk , we won't use this stream
continue
}
const parent = await openDiskChain({
handler: this._adapter._handler,
path: vhdPath,
})
let disposableDescendants
const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
// this will also clean if another disk of this VM backup fails
// if user really only need to restore non failing disks he can retry with ignoredVdis
@@ -135,7 +132,8 @@ export class ImportVmBackup {
if (!disposed) {
disposed = true
try {
await parent?.close()
await disposableDescendants?.dispose()
await disposableSynthetic?.dispose()
} catch (error) {
warn('openVhd: failed to dispose VHDs', { error })
}
@@ -143,10 +141,11 @@ export class ImportVmBackup {
}
$defer.onFailure(() => disposeOnce())
debug('got vhd synthetic of parents', parent)
const parentVhd = disposableSynthetic.value
await parentVhd.readBlockAllocationTable()
debug('got vhd synthetic of parents', parentVhd.length)
if (snapshotCandidate !== undefined) {
let descendant, negativeDisk
try {
debug('will try to use differential restore', {
backupWithSnapshotPath,
@@ -154,37 +153,39 @@ export class ImportVmBackup {
vdiRef,
})
descendant = await openDiskChain({
handler: this._adapter._handler,
path: backupWithSnapshotPath,
disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
until: vhdPath,
})
const descendantsVhd = disposableDescendants.value
await descendantsVhd.readBlockAllocationTable()
debug('got vhd synthetic of descendants')
negativeDisk = new NegativeDisk(parent, descendant)
const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
debug('got vhd negative')
// update the stream with the negative vhd stream
disk = negativeDisk
stream = await negativeVhd.stream()
vdis[vdiRef].baseVdi = snapshotCandidate
} catch (error) {
// can be a broken VHD chain, a vhd chain with a key backup, ....
// not an irrecuperable error, don't dispose parentVhd, and fall back to full restore
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
warn(`can't use differential restore`, { error })
descendant?.close()
negativeDisk?.close()
disposableDescendants?.dispose()
}
}
// didn't make a negative stream : fall back to classic stream
if (disk === undefined) {
// didn't make a negative stream : fallback to classic stream
if (stream === undefined) {
debug('use legacy restore')
disk = parent
stream = await parentVhd.stream()
}
info('everything is ready, will transfer', disk)
disks[vdiRef] = disk
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
info('everything is ready, will transfer', stream.length)
streams[`${vdiRef}.vhd`] = stream
}
return {
disks,
streams,
vbds,
vdis,
version: '1.0.0',
@@ -195,7 +196,7 @@ export class ImportVmBackup {
}
async #decorateIncrementalVmMetadata() {
const { additionalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
const ignoredVdis = new Set(
Object.entries(mapVdisSrs)
@@ -212,8 +213,8 @@ export class ImportVmBackup {
const cache = new Map()
const mapVdisSrRefs = {}
if (additionalVmTag !== undefined) {
backup.vm.tags.push(additionalVmTag)
if (additionnalVmTag !== undefined) {
backup.vm.tags.push(additionnalVmTag)
}
for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
@@ -240,6 +241,7 @@ export class ImportVmBackup {
assert.strictEqual(metadata.mode, 'delta')
backup = await this.#decorateIncrementalVmMetadata()
Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
}
return Task.run(
@@ -255,14 +257,7 @@ export class ImportVmBackup {
: await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
newMacAddresses,
})
const remoteName = adapter._handler._remote.name
let desc = `Restored on ${formatFilenameDate(+new Date())}`
if (remoteName !== undefined) {
desc += ` from ${remoteName}`
}
if (metadata.vm.name_description) {
desc += ` - ${metadata.vm.name_description}`
}
await Promise.all([
xapi.call('VM.add_tags', vmRef, 'restored from backup'),
xapi.call(
@@ -270,8 +265,13 @@ export class ImportVmBackup {
vmRef,
`${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
),
xapi.call('VM.set_name_description', vmRef, desc),
resetVmOtherConfig(xapi, vmRef),
xapi.call(
'VM.set_name_description',
vmRef,
`Restored on ${formatFilenameDate(+new Date())} from ${adapter._handler._remote.name} -
${metadata.vm.name_description}
`
),
])
return {

View File

@@ -2,7 +2,7 @@ import { asyncEach } from '@vates/async-each'
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import { compose } from '@vates/compose'
import { createLogger } from '@xen-orchestra/log'
import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
import { decorateMethodsWith } from '@vates/decorate-with'
import { deduped } from '@vates/disposable/deduped.js'
import { dirname, join, resolve } from 'node:path'
@@ -10,6 +10,7 @@ import { execFile } from 'child_process'
import { mount } from '@vates/fuse-vhd'
import { readdir, lstat } from 'node:fs/promises'
import { synchronized } from 'decorator-synchronized'
import { v4 as uuidv4 } from 'uuid'
import { ZipFile } from 'yazl'
import Disposable from 'promise-toolbox/Disposable'
import fromCallback from 'promise-toolbox/fromCallback'
@@ -18,7 +19,7 @@ import groupBy from 'lodash/groupBy.js'
import pDefer from 'promise-toolbox/defer'
import pickBy from 'lodash/pickBy.js'
import reduce from 'lodash/reduce.js'
import * as tar from 'tar'
import tar from 'tar'
import zlib from 'zlib'
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
@@ -27,24 +28,19 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { getTmpDir } from './_getTmpDir.mjs'
import { isMetadataFile } from './_backupType.mjs'
import { isValidXva } from './_isValidXva.mjs'
import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
import { lvs, pvs } from './_lvm.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
import { openDiskChain } from './disks/openDiskChain.mjs'
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
import { ReadAhead } from '@xen-orchestra/disk-transform'
export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
const compareTimestamp = (a, b) => a.timestamp - b.timestamp
const noop = Function.prototype
@@ -205,7 +201,7 @@ export class RemoteAdapter {
})
}
// check if we will be allowed to merge a vhd created in this adapter
// check if we will be allowed to merge a a vhd created in this adapter
// with the vhd at path `path`
async isMergeableParent(packedParentUid, path) {
return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
@@ -312,8 +308,8 @@ export class RemoteAdapter {
}
async deleteVmBackups(files) {
const metadata = await asyncMap(files, file => this.readVmBackupMetadata(file))
const { delta, full, ...others } = groupBy(metadata, 'mode')
const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
const { delta, full, ...others } = groupBy(metadatas, 'mode')
const unsupportedModes = Object.keys(others)
if (unsupportedModes.length !== 0) {
@@ -427,20 +423,12 @@ export class RemoteAdapter {
async listAllVms() {
const handler = this._handler
const vmsUuids = []
try {
await asyncEach(await handler.list(BACKUP_DIR), async entry => {
// ignore hidden and lock files
if (entry[0] !== '.' && !entry.endsWith('.lock')) {
vmsUuids.push(entry)
}
})
} catch (error) {
// remote without any VM backup are ok
if (error.code !== 'ENOENT') {
throw error
await asyncEach(await handler.list(BACKUP_DIR), async entry => {
// ignore hidden and lock files
if (entry[0] !== '.' && !entry.endsWith('.lock')) {
vmsUuids.push(entry)
}
}
})
return vmsUuids
}
@@ -498,7 +486,7 @@ export class RemoteAdapter {
const results = []
await asyncMapSettled(partitions, partition =>
partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
partition.type === LVM_PARTITION_TYPE
? this._listLvmLogicalVolumes(devicePath, partition, results)
: results.push(partition)
)
@@ -575,7 +563,7 @@ export class RemoteAdapter {
}
}
async #getCacheableDataListVmBackups(dir) {
async #getCachabledDataListVmBackups(dir) {
debug('generating cache', { path: dir })
const handler = this._handler
@@ -613,11 +601,6 @@ export class RemoteAdapter {
// if cache is missing or broken => regenerate it and return
async _readCacheListVmBackups(vmUuid) {
// immutable remote can't use any caching
// since the cache file may be non modifiable
if (this._handler.isImmutable()) {
return this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
}
const path = this.#getVmBackupsCache(vmUuid)
const cache = await this._readCache(path)
@@ -627,7 +610,7 @@ export class RemoteAdapter {
}
// nothing cached, or cache unreadable => regenerate it
const backups = await this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
if (backups === undefined) {
return
}
@@ -698,32 +681,26 @@ export class RemoteAdapter {
return path
}
async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
const handler = this._handler
if (this.useVhdDirectory()) {
await writeToVhdDirectory({
disk,
target: {
handler,
path,
concurrency: writeBlockConcurrency,
validator,
compression: 'brotli',
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
concurrency: writeBlockConcurrency,
compression: this.#getCompressionType(),
async validator() {
await input.task
return validator.apply(this, arguments)
},
})
await VhdAbstract.createAlias(handler, path, dataPath)
return size
} else {
const stream = await toVhdStream(disk)
await this.outputStream(path, stream, { validator, checksum: false })
await validator(path)
return this.outputStream(path, input, { checksum, validator })
}
}
async outputStream(
path,
input,
{ checksum = !this._handler.isEncrypted, maxStreamLength, streamLength, validator = noop } = {}
) {
async outputStream(path, input, { checksum = true, maxStreamLength, streamLength, validator = noop } = {}) {
const container = watchStreamSize(input)
await this._handler.outputStream(path, input, {
checksum,
@@ -739,16 +716,30 @@ export class RemoteAdapter {
}
// open the hierarchy of ancestors until we find a full one
async _createVhdDisk(handler, path, { useChain }) {
let disk
if (useChain) {
disk = await openDiskChain({ handler, path })
} else {
disk = new RemoteVhdDisk({ handler, path })
await disk.init()
async _createVhdStream(handler, path, { useChain }) {
const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
// I don't want the vhds to be disposed on return
// but only when the stream is done ( or failed )
let disposed = false
const disposeOnce = async () => {
if (!disposed) {
disposed = true
try {
await disposableSynthetic.dispose()
} catch (error) {
warn('openVhd: failed to dispose VHDs', { error })
}
}
}
disk = new ReadAhead(disk)
return disk
const synthetic = disposableSynthetic.value
await synthetic.readBlockAllocationTable()
const stream = await synthetic.stream()
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
return stream
}
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
@@ -756,14 +747,14 @@ export class RemoteAdapter {
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
const dir = dirname(metadata._filename)
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
const disks = {}
const streams = {}
await asyncMapSettled(Object.keys(vdis), async ref => {
delete vdis[ref].baseVdi
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
})
return {
disks,
streams,
vbds,
vdis,
version: '1.0.0',
@@ -781,10 +772,10 @@ export class RemoteAdapter {
let json
let isImmutable = false
let remoteIsImmutable = false
// if the remote is immutable, check if this metadata is also immutable
// if the remote is immutable, check if this metadatas are also immutables
try {
// this file is not encrypted
await this._handler._readFile(IMMUTABILITY_METADATA_FILENAME)
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
remoteIsImmutable = true
} catch (error) {
if (error.code !== 'ENOENT') {
@@ -797,7 +788,7 @@ export class RemoteAdapter {
json = await this.handler.readFile(path, { flag: 'r+' })
// s3 handler don't respect flags
} catch (err) {
// retry without triggering immutability check ,only on immutable remote
// retry without triggerring immutbaility check ,only on immutable remote
if (err.code === 'EPERM' && remoteIsImmutable) {
isImmutable = true
json = await this._handler.readFile(path, { flag: 'r' })

View File

@@ -24,7 +24,7 @@ export class RestoreMetadataBackup {
const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
const data = await handler.readFile(dataFileName)
// if data is JSON, sent it as a plain string; otherwise, consider the data as binary and encode it
// if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
const isJson = dataFileName.endsWith('.json')
return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
}

View File

@@ -1,5 +1,4 @@
export const isMetadataFile = filename => filename.endsWith('.json')
export const isVhdFile = filename => filename.endsWith('.vhd')
export const isVhdSumFile = filename => filename.endsWith('.vhd.checksum')
export const isXvaFile = filename => filename.endsWith('.xva')
export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')

View File

@@ -83,12 +83,12 @@ test('It remove broken vhd', async () => {
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
let loggued = ''
const logInfo = message => {
logged += message
loggued += message
}
await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
assert.equal(logged, `VHD check error`)
assert.equal(loggued, `VHD check error`)
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
@@ -98,7 +98,7 @@ test('It remove broken vhd', async () => {
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent, should be deleted
await generateVhd(`${basePath}/abandoned.vhd`, {
await generateVhd(`${basePath}/abandonned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
@@ -118,21 +118,21 @@ test('it remove vhd with missing or multiple ancestors', async () => {
`metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`],
}),
{ flags: 'w' }
)
// clean
let logged = ''
let loggued = ''
const logInfo = message => {
logged += message + '\n'
loggued += message + '\n'
}
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
// we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
})
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
@@ -144,12 +144,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
vhds: [
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.json is not here
// abandonned.json is not here
],
})
)
await generateVhd(`${basePath}/abandoned.vhd`)
await generateVhd(`${basePath}/abandonned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
@@ -162,12 +162,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
},
})
let logged = ''
let loggued = ''
const logInfo = message => {
logged += message + '\n'
loggued += message + '\n'
}
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
let matched = logged.match(/deleting unused VHD/g) || []
let matched = loggued.match(/deleting unused VHD/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
@@ -179,14 +179,14 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
`deleted.vhd`, // in metadata but not in vhds
`orphan.vhd`,
`child.vhd`,
// abandoned.vhd is not here anymore
// abandonned.vhd is not here anymore
],
}),
{ flags: 'w' }
)
logged = ''
loggued = ''
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
matched = logged.match(/deleting unused VHD/g) || []
matched = loggued.match(/deleting unused VHD/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
})
@@ -221,21 +221,21 @@ test('it merges delta of non destroyed chain', async () => {
},
})
let logged = []
let loggued = []
const logInfo = message => {
logged.push(message)
loggued.push(message)
}
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
assert.equal(logged[0], `unexpected number of entries in backup cache`)
assert.equal(loggued[0], `unexpected number of entries in backup cache`)
logged = []
loggued = []
await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
const [merging] = logged
const [merging] = loggued
assert.equal(merging, `merging VHD chain`)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children after the merge
assert.equal(metadata.size, 104448)
assert.equal(metadata.size, 104960)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
@@ -286,57 +286,10 @@ test('it finish unterminated merge ', async () => {
assert.equal(remainingVhds.includes('child.vhd'), true)
})
test('it finishes unterminated merge when child is a disk chain', async () => {
// This test covers the case where a merge was interrupted and the child disk is a disk chain.
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 209920,
vhds: [`${relativePath}/child.vhd`],
})
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
header: {
parentUnicodeName: 'ancestor.vhd',
parentUuid: ancestor.footer.uuid,
},
})
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'intermediate.vhd',
parentUuid: intermediate.footer.uuid,
},
})
// merge state file with chain field
await handler.writeFile(
`${basePath}/.ancestor.vhd.merge.json`,
JSON.stringify({
parent: { uuid: '0' },
child: { uuid: '0' },
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
currentBlock: 0,
mergedDataSize: 0,
step: 'mergeBlocks',
diskSize: 0,
})
)
await adapter.cleanVm(rootPath, { remove: true, merge: true, logWarn: () => {}, lock: false })
// ancestor should be renamed to child.vhd (replacing it), intermediate.vhd should be deleted
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 1)
assert.equal(remainingVhds[0], 'child.vhd')
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
describe('tests multiple combination ', { concurrency: 1 }, () => {
// the message an resulting files should be identical to the output with vhd files which is tested independantly
describe('tests multiple combination ', () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
@@ -356,10 +309,10 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
// a vhd non referenced in metada
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
await generateVhd(`${basePath}/abandoned.vhd`, {
// an abandonded delta vhd without its parent
await generateVhd(`${basePath}/abandonned.vhd`, {
useAlias,
mode: vhdMode,
header: {
@@ -450,12 +403,13 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 4404224)
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
// console.log(survivors)
if (useAlias) {
const dataSurvivors = await handler.list(basePath + '/data')
// the goal of the alias : do not move a full folder
@@ -477,6 +431,7 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
@@ -501,7 +456,7 @@ test('check Aliases should work alone', async () => {
logWarn: () => {},
})
// only ok have survived
// only ok have suvived
const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
assert.equal(alias.length, 1)

View File

@@ -1,13 +1,12 @@
import * as UUID from 'uuid'
import sum from 'lodash/sum.js'
import { asyncMap } from '@xen-orchestra/async-map'
import { Constants, openVhd, VhdAbstract } from 'vhd-lib'
import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
import { basename, dirname, resolve } from 'node:path'
import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
import { limitConcurrency } from 'limit-concurrency-decorator'
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
import { mergeVhdChain } from 'vhd-lib/merge.js'
import { Task } from './Task.mjs'
import { Disposable } from 'promise-toolbox'
@@ -15,6 +14,27 @@ import handlerPath from '@xen-orchestra/fs/path'
const { DISK_TYPES } = Constants
// checking the size of a vhd directory is costly
// 1 Http Query per 1000 blocks
// we only check size of all the vhd are VhdFiles
function shouldComputeVhdsSize(handler, vhds) {
if (handler.isEncrypted) {
return false
}
return vhds.every(vhd => vhd instanceof VhdFile)
}
const computeVhdsSize = (handler, vhdPaths) =>
Disposable.use(
vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
async vhds => {
if (shouldComputeVhdsSize(handler, vhds)) {
const sizes = await asyncMap(vhds, vhd => vhd.getSize())
return sum(sizes)
}
}
)
// chain is [ ancestor, child_1, ..., child_n ]
async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
logInfo(`merging VHD chain`, { chain })
@@ -31,15 +51,7 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
}
}, 10e3)
try {
const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
const childDisks = []
for (const path of chain) {
childDisks.push(new RemoteVhdDisk({ handler, path }))
}
const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
return await mergeVhdChain(handler, chain, {
logInfo,
mergeBlockConcurrency,
onProgress({ done: d, total: t }) {
@@ -48,14 +60,6 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
},
removeUnused: remove,
})
const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
await parentDisk.init({ force: isResumingMerge })
await childDiskChain.init({ force: isResumingMerge })
const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
return result
} finally {
clearInterval(handle)
}
@@ -64,12 +68,10 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
const noop = Function.prototype
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
const listVhds = async (handler, vmDir, logWarn) => {
const vhds = new Set()
const aliases = {}
const interruptedVhds = new Map()
const checksums = new Set()
await asyncMap(
await handler.list(`${vmDir}/vdis`, {
@@ -83,15 +85,11 @@ const listVhds = async (handler, vmDir, logWarn) => {
}),
async vdiDir => {
const list = await handler.list(vdiDir, {
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
})
aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
await asyncMap(list, async file => {
if (isVhdSumFile(file)) {
checksums.add(`${vdiDir}/${file}`)
return
}
const res = INTERRUPTED_VHDS_REG.exec(file)
if (res === null) {
vhds.add(`${vdiDir}/${file}`)
@@ -113,7 +111,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
)
)
return { vhds, interruptedVhds, aliases, checksums }
return { vhds, interruptedVhds, aliases }
}
export async function checkAliases(
@@ -131,29 +129,10 @@ export async function checkAliases(
logWarn('missing target of alias', { alias })
if (remove) {
logInfo('removing alias and non VHD target', { alias, target })
await handler.unlink(target)
await handler.unlink(alias)
}
continue
}
if (err.code === 'EISDIR') {
logWarn('alias is a vhd directory', { alias })
if (remove) {
logInfo('removing vhd directory named as alias', { alias, target })
await VhdAbstract.unlink(handler, alias)
}
continue
}
logWarn('unhandled error while checking alias', { alias, err })
continue
}
if (target === '') {
logWarn('empty target for alias ', { alias })
if (remove) {
logInfo('removing alias and non VHD target', { alias, target })
await handler.unlink(alias)
}
continue
}
if (!isVhdFile(target)) {
@@ -230,16 +209,7 @@ export async function cleanVm(
const vhdParents = { __proto__: null }
const vhdChildren = { __proto__: null }
const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
// from 5.110 to 5.113 we computed checksum for vhd file
// but never used nor removed them
await asyncMap(checksums, async path => {
if (remove) {
logInfo('deleting checksum file ', { path })
return handler.unlink(path)
}
})
const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
// remove broken VHDs
await asyncMap(vhds, async path => {
@@ -517,14 +487,14 @@ export async function cleanVm(
const metadataWithMergedVhd = {}
const doMerge = async () => {
await asyncMap(toMerge, async chain => {
const { finalDiskSize } = await limitedMergeVhdChain(handler, chain, {
const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
logInfo,
logWarn,
remove,
mergeBlockConcurrency,
})
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
})
}
@@ -558,7 +528,7 @@ export async function cleanVm(
let fileSystemSize
const mergedSize = metadataWithMergedVhd[metadataPath]
const { mode, size, xva } = metadata
const { mode, size, vhds, xva } = metadata
try {
if (mode === 'full') {
@@ -574,7 +544,21 @@ export async function cleanVm(
})
}
} catch (error) {
// will fail with encrypted remote
// can fail with encrypted remote
}
} else if (mode === 'delta') {
// don't warn if the size has changed after a merge
if (mergedSize === undefined) {
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
// the size is not computed in some cases (e.g. VhdDirectory)
if (fileSystemSize !== undefined && fileSystemSize !== size) {
logWarn('cleanVm: incorrect backup size in metadata', {
path: metadataPath,
actual: size ?? 'none',
expected: fileSystemSize,
})
}
}
}
} catch (error) {
@@ -591,8 +575,8 @@ export async function cleanVm(
if (mergedSize) {
// all disks are now key disk
metadata.isVhdDifferencing = {}
for (const id of Object.keys(metadata.vdis ?? {})) {
metadata.isVhdDifferencing[id] = false
for (const id of Object.values(metadata.vdis ?? {})) {
metadata.isVhdDifferencing[`${id}.vhd`] = false
}
}
mustRegenerateCache = true

View File

@@ -1,58 +1,5 @@
// @ts-check
import moment from 'moment-timezone'
import assert from 'node:assert'
/**
* @typedef {import('moment-timezone').Moment} Moment
*/
/**
* @typedef {Object} LTRSettings
* @property {number} [firstHourOfTheDay] - First hour of the day (for daily retention)
* @property {number} [firstDayOfWeek] - First day of the week (for weekly retention)
* @property {number} [firstDayOfMonth] - First day of the month (for monthly retention)
* @property {number} [firstDayOfYear] - First day of the year (for yearly retention)
*/
/**
* @typedef {Object} LTRConfig
* @property {number} retention - Number of time buckets to retain
* @property {LTRSettings} [settings] - Optional settings for the retention period
*/
/**
* @typedef {Object} LongTermRetention
* @property {LTRConfig} [daily] - Daily retention configuration
* @property {LTRConfig} [weekly] - Weekly retention configuration
* @property {LTRConfig} [monthly] - Monthly retention configuration
* @property {LTRConfig} [yearly] - Yearly retention configuration
*/
/**
* @typedef {Object} Entry
* @property {number | undefined} timestamp - Unix timestamp of the entry
* @property {string} id
* @property {*} [key] - Any other properties
*/
/**
* @typedef {Object} DateBucket
* @property {number} remaining - Number of remaining buckets to fill
* @property {string|null} lastMatchingBucket - Last bucket key that was matched
* @property {(date: Moment) => string} formatter - Function to format date into bucket key
* @property {Object.<string, Entry>} entries - Map of bucket keys to entries
*/
/**
* @typedef {Object} LTRDefinition
* @property {(options: {firstHourOfTheDay?: number, firstDayOfWeek?: number, firstDayOfMonth?: number, firstDayOfYear?: number, dateCreator: (date: Moment) => Moment}) => (date: Moment) => string} makeDateFormatter
* @property {string} [ancestor] - Name of the parent retention period
*/
/**
* Creates a function that converts dates to moment objects in a specific timezone
* @param {string|undefined} timezone - IANA timezone string (e.g., 'America/New_York')
* @returns {(date: Moment | number) => Moment} Function that creates timezoned moment objects
*/
function instantiateTimezonedDateCreator(timezone) {
return date => {
@@ -62,12 +9,9 @@ function instantiateTimezonedDateCreator(timezone) {
}
}
/**
* @type {Object.<string, LTRDefinition>}
*/
const LTR_DEFINITIONS = {
daily: {
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator }) => {
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
return date => {
const copy = dateCreator(date)
copy.hour(copy.hour() - firstHourOfTheDay)
@@ -76,20 +20,20 @@ const LTR_DEFINITIONS = {
},
},
weekly: {
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator }) => {
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
return date => {
const copy = dateCreator(date)
copy.date(copy.date() - firstDayOfWeek)
copy.date(date.date() - firstDayOfWeek)
// warning, the year in term of week may different from YYYY
// since the computation of the first week of a year is timezone dependent
return copy.format('gggg-ww')
// since the computation of the first week of a year is timezone dependant
return copy.format('gggg-WW')
}
},
ancestor: 'daily',
},
monthly: {
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator }) => {
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
return date => {
const copy = dateCreator(date)
copy.date(copy.date() - firstDayOfMonth)
@@ -99,7 +43,7 @@ const LTR_DEFINITIONS = {
ancestor: 'weekly',
},
yearly: {
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator }) => {
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
return date => {
const copy = dateCreator(date)
copy.date(copy.date() - firstDayOfYear)
@@ -109,23 +53,27 @@ const LTR_DEFINITIONS = {
ancestor: 'monthly',
},
}
/**
* Groups entries into date buckets based on long-term retention configuration
* @template {Entry} T
* @param {Array<T>} entries - Array of entries sorted in descending order by timestamp
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
* @param {string | undefined} timezone - IANA timezone string
* @returns {Object.<string, DateBucket>} Map of retention period names to their date buckets
*/
export function getLtrEntries(entries, longTermRetention, timezone) {
/**
* @type {Object.<string, DateBucket>}
*/
// returns all entries but the last retention-th
/**
* return the entries too old to be kept
* if multiple entries are i the same time bucket : keep only the most recent one
* if an entry is valid in any of the bucket OR the minRetentionCount : keep it
* if a bucket is completly empty : it does not count as one, thus it may extend the retention
* @returns Array<Backup>
*/
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
assert.strictEqual(
typeof minRetentionCount,
'number',
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
)
assert.strictEqual(
minRetentionCount >= 0,
true,
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
)
const dateBuckets = {}
if (!longTermRetention || Object.keys(longTermRetention).length === 0) {
return {}
}
const dateCreator = instantiateTimezonedDateCreator(timezone)
// only check buckets that have a retention set
for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
@@ -133,7 +81,7 @@ export function getLtrEntries(entries, longTermRetention, timezone) {
assert.notStrictEqual(
timezone,
undefined,
`timezone must defined for ltr, got ${JSON.stringify({ longTermRetention, timezone })}`
`timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
)
assert.strictEqual(
typeof retention,
@@ -149,106 +97,40 @@ export function getLtrEntries(entries, longTermRetention, timezone) {
remaining: retention,
lastMatchingBucket: null,
formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
entries: {}, // bucket => entry id
}
}
const nb = entries.length
let previousTimestamp = -1
const oldEntries = []
for (let i = nb - 1; i >= 0; i--) {
const entry = entries[i]
// force it so the compiler is sure we have a timestamp
if (entry.timestamp === undefined) {
assert.notStrictEqual(
entry.timestamp,
undefined,
`Can't compute long term retention if entries don't have a timestamp`
)
continue
}
// we go through the entries from the last (most recent) to the first (oldest)
assert.ok(
previousTimestamp === -1 || entry.timestamp < previousTimestamp,
`entries must be sorted in desc order ${new Date(entry.timestamp)} , previous : > ${new Date(previousTimestamp)} `
)
previousTimestamp = entry.timestamp
const entryDate = dateCreator(entry.timestamp)
let shouldBeKept = false
for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
if (remaining === 0) {
continue
}
const bucket = formatter(entryDate)
if (lastMatchingBucket !== bucket) {
if (remaining === 0) {
continue
if (lastMatchingBucket !== null) {
assert.strictEqual(
lastMatchingBucket > bucket,
true,
`entries must be sorted in asc order ${lastMatchingBucket} ${bucket}`
)
}
dateBuckets[duration].lastMatchingBucket = bucket
shouldBeKept = true
dateBuckets[duration].remaining -= 1
}
dateBuckets[duration].entries[bucket] = entry
}
}
return dateBuckets
}
/**
* return the status of an entry
* @param {Array<Entry>} entries
* @param {Entry} entry
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
* @param {string | undefined} timezone - IANA timezone string
* @returns {Array<{duration: string, dateBucket:string}>}
*/
export function getEntryStatus(entries, entry, longTermRetention, timezone) {
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
const entryBuckets = []
for (const [duration, { entries }] of Object.entries(dateBuckets)) {
for (const [dateBucket, bucketEntry] of Object.entries(entries)) {
if (entry.id === bucketEntry.id) {
entryBuckets.push({ duration, dateBucket })
dateBuckets[duration].lastMatchingBucket = bucket
}
}
}
return entryBuckets
}
/**
* return the entries too old to be kept
* if multiple entries are in the same time bucket: keep only the oldest one
* if an entry is valid in any of the bucket OR the minRetentionCount: keep it
* if a bucket is completely empty: it does not count as one, thus it may extend the retention
* if an entry is within the most recents minRetentionCount entries : keep it
*
* @param {number} minRetentionCount - Minimum number of most recent entries to always keep
* @param {Array<Entry>} entries - Array of entries sorted in descending order by timestamp
* @param {Object} options - Additional options
* @param {LongTermRetention} [options.longTermRetention={}] - Configuration for retention periods
* @param {string | undefined} [options.timezone] - IANA timezone string
* @returns {Array<Entry>} Array of entries that can be removed
*/
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
assert.strictEqual(
typeof minRetentionCount,
'number',
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
)
assert.ok(
minRetentionCount >= 0,
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
)
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
const kept = new Set(entries.filter((_, index) => index >= entries.length - minRetentionCount))
/**
* @type {Set<Entry>}
*/
for (const { entries } of Object.values(dateBuckets)) {
for (const entry of Object.values(entries)) {
kept.add(entry)
if (i >= nb - minRetentionCount) {
shouldBeKept = true
}
if (!shouldBeKept) {
oldEntries.push(entry)
}
}
// ensure order is the same as the source
const oldEntries = entries.filter((entry, index) => {
return !kept.has(entry)
})
return oldEntries
// we expect the entries to be in the right order
return oldEntries.reverse()
}

View File

@@ -10,28 +10,24 @@ describe('_getOldEntries() should succeed', () => {
1,
[
{ timestamp: 1, id: 1 },
{ timestamp: 2, id: 2 },
{ timestamp: 3, id: 3 },
{ timestamp: 3, id: 2 },
{ timestamp: 2, id: 3 },
],
],
expectedIds: [1, 2],
testLabel: 'should handle number based retention ',
},
{
args: [3, [{ id: 1 }, 4, new Date(), 'whatever']],
expectedIds: [1],
testLabel: 'should handle number based retention without timestamp ',
},
{
args: [
0,
[
{ timestamp: +new Date('2024-09-01 00:00:00'), id: 1 }, // too old
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 2 }, // too old
{ timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // oldest in same day
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // too old
{ timestamp: +new Date('2024-09-01 00:00:00'), id: 2 }, // too old
{ timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // too old in same day
{ timestamp: +new Date('2024-09-02 00:10:00'), id: 4 },
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 5 },
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // oldest in same day
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // too old in same day
{ timestamp: +new Date('2024-09-04 00:10:00'), id: 7 },
],
{
@@ -41,22 +37,22 @@ describe('_getOldEntries() should succeed', () => {
timezone: 'Europe/Paris',
},
],
expectedIds: [1, 2, 4, 7],
expectedIds: [1, 2, 3, 6],
testLabel: 'should handle day based retention ',
},
{
args: [
0,
[
{ timestamp: +new Date('2024-08-30 00:01:00'), id: 1 }, // week n-3 too old
{ timestamp: +new Date('2024-08-31 00:00:00'), id: 2 }, // week n-3 too old
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2, oldest
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // week n-3 too old
{ timestamp: +new Date('2024-09-02 00:00:00'), id: 2 }, // week n-3 too old
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 4 }, // week n-2
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // week n-2
{ timestamp: +new Date('2024-09-06 00:09:00'), id: 6 }, // week n-2
{ timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2
{ timestamp: +new Date('2024-09-15 00:09:00'), id: 8 }, // week n-1, oldest
{ timestamp: +new Date('2024-09-21 00:09:00'), id: 9 }, // week n-1 most recent
{ timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2, most recent kept
{ timestamp: +new Date('2024-09-09 00:09:00'), id: 8 }, // week n-1, too old
{ timestamp: +new Date('2024-09-15 00:09:00'), id: 9 }, // week n-1 kept
{ timestamp: +new Date('2024-09-22 00:09:00'), id: 10 }, // week n kept
],
{
@@ -66,7 +62,7 @@ describe('_getOldEntries() should succeed', () => {
timezone: 'Europe/Paris',
},
],
expectedIds: [1, 2, 4, 5, 6, 7, 9],
expectedIds: [1, 2, 3, 4, 5, 6, 8],
testLabel: 'should handle week based retention ',
},
{
@@ -89,26 +85,26 @@ describe('_getOldEntries() should succeed', () => {
timezone: 'Europe/Paris',
},
],
expectedIds: [1, 4, 5, 7, 8],
expectedIds: [1, 3, 4, 6, 7],
testLabel: 'should handle month based retention',
},
{
args: [
2,
0,
[
{ timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, //
{ timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // year n -1
{ timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, //
{ timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, //
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, //
{ timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, // month n-4
{ timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, // month n-3
{ timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, // week n-3
{ timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, // week n-2, month n-2,
{ timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, // week n-1, year n
{ timestamp: +new Date('2025-01-02 00:10:00'), id: 11 }, // day n-1, month n-1,
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // day n , week n , month n
{ timestamp: +new Date('2025-12-31 10:09:00'), id: 13 }, // min retention
{ timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, // too old
{ timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // too old in same year
{ timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, // too old
{ timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, // too old
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // too old
{ timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, // new month,
{ timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, // new month , week reached retention
{ timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, // new week
{ timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, // new week/month / year daily reach retention
{ timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, // same day/week/month/year
{ timestamp: +new Date('2025-01-01 00:10:00'), id: 11 }, // new day / week / month
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // same day/week/month/year
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 13 }, // new month /year
],
{
longTermRetention: {
@@ -120,18 +116,18 @@ describe('_getOldEntries() should succeed', () => {
timezone: 'Europe/Paris', // use a time zone here because week definition is timezone bound
},
],
expectedIds: [1, 3, 4, 5],
expectedIds: [1, 2, 3, 4, 5, 10, 12],
testLabel: 'complete test ',
},
{
args: [
0,
[
{ timestamp: +new Date('2024-08-31 09:01:00'), id: 1 }, // 4th day too old
{ timestamp: +new Date('2024-09-01 00:10:00'), id: 2 }, // third day
{ timestamp: +new Date('2024-09-01 14:20:00'), id: 3 }, // second day, oldest
{ timestamp: +new Date('2024-09-02 00:22:00'), id: 4 }, // second day
{ timestamp: +new Date('2024-09-03 14:00:00'), id: 5 }, // same day oldest
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // thrid day too old
{ timestamp: +new Date('2024-09-01 00:10:00'), id: 2 }, // thrid day
{ timestamp: +new Date('2024-09-01 00:20:00'), id: 3 }, // second day, too old
{ timestamp: +new Date('2024-09-02 00:22:00'), id: 4 }, // second day too old
{ timestamp: +new Date('2024-09-03 00:20:00'), id: 5 }, // second day in NZ
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // same day in NZ
{ timestamp: +new Date('2024-09-04 00:10:00'), id: 7 }, // most recent
],
@@ -139,10 +135,10 @@ describe('_getOldEntries() should succeed', () => {
longTermRetention: {
daily: { retention: 3 },
},
timezone: 'Pacific/Auckland', // GMT +13
timezone: 'Pacific/Auckland', // GMT +6
},
],
expectedIds: [1, 4, 6, 7],
expectedIds: [1, 2, 3, 6],
testLabel: 'should handle timezone ',
},
]
@@ -153,20 +149,13 @@ describe('_getOldEntries() should succeed', () => {
assert.strictEqual(
oldEntries.length,
expectedIds.length,
`different length , ${JSON.stringify({ expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
`different length , ${JSON.stringify({ oldEntries, expectedIds })}`
)
for (let i = 0; i < expectedIds.length; i++) {
if (i > 0) {
assert.strictEqual(
expectedIds[i - 1] < expectedIds[i],
true,
`expect ids in ascending order ${expectedIds[i - 1]} ${expectedIds[i]}`
)
}
assert.strictEqual(
oldEntries[i].id,
expectedIds[i],
`different id , ${JSON.stringify({ i, expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
`different id , ${JSON.stringify({ i, expectedIds, oldEntries })}`
)
}
})
@@ -283,19 +272,6 @@ describe('_getOldEntries() should fail when called incorrectly', () => {
],
testLabel: 'broken date ',
},
{
args: [
1,
[{ obj: 1 }, 'toto', Date.now()],
{
longTermRetention: {
daily: { retention: 5 },
},
timezone: 'Europe/Paris',
},
],
testLabel: 'should fail if LTR without timestamp',
},
]
for (const { args, testLabel } of tests) {

View File

@@ -8,11 +8,7 @@ import { defer } from 'golike-defer'
import { cancelableMap } from './_cancelableMap.mjs'
import { Task } from './Task.mjs'
import pick from 'lodash/pick.js'
import { BASE_DELTA_VDI, CONTENT_KEY, COPY_OF, VM_UUID } from './_otherConfig.mjs'
import { VHD_MAX_SIZE, XapiDiskSource } from '@xen-orchestra/xapi'
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
import { toQcow2Stream } from '@xen-orchestra/qcow2'
import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
@@ -23,7 +19,7 @@ export async function exportIncrementalVm(
) {
// refs of VM's VDIs → base's VDIs.
const disks = {}
const streams = {}
const vdis = {}
const vbds = {}
await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
@@ -56,14 +52,32 @@ export async function exportIncrementalVm(
$snapshot_of$uuid: vdi.$snapshot_of?.uuid,
$SR$uuid: vdi.$SR.uuid,
}
disks[vdiRef] = new XapiDiskSource({
vdiRef,
xapi: vm.$xapi,
baseRef: baseVdi?.$ref,
nbdConcurrency,
preferNbd,
})
await disks[vdiRef].init()
try {
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
baseRef: baseVdi?.$ref,
cancelToken,
format: 'vhd',
nbdConcurrency,
preferNbd,
})
} catch (err) {
if (err.code === 'VDI_CANT_DO_DELTA') {
// fall back to a base
Task.info(`Can't do delta, will try to get a full stream`, { vdi })
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
cancelToken,
format: 'vhd',
nbdConcurrency,
preferNbd,
})
// only warn if the fall back succeed
Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
vdi,
})
} else {
throw err
}
}
})
const suspendVdi = vm.$suspend_VDI
@@ -73,13 +87,10 @@ export async function exportIncrementalVm(
...suspendVdi,
$SR$uuid: suspendVdi.$SR.uuid,
}
disks[vdiRef] = new XapiDiskSource({
vdiRef: suspendVdi.$ref,
xapi: vm.$xapi,
nbdConcurrency,
preferNbd,
streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
cancelToken,
format: 'vhd',
})
await disks[vdiRef].init()
}
const vifs = {}
@@ -93,13 +104,11 @@ export async function exportIncrementalVm(
}
})
// Get a fresh list of VM's VTPM to avoid `vm.VTPMs: [undefined]`
const vmVtpms = await vm.$xapi.getField('VM', vm.$ref, 'VTPMs')
const vtpms = await Promise.all(
vmVtpms.map(async vtpmRef => {
vm.$VTPMs.map(async vtpm => {
let content
try {
content = await vm.$xapi.call('VTPM.get_contents', vtpmRef)
content = await vm.$xapi.call('VTPM.get_contents', vtpm.$ref)
} catch (err) {
console.error(err)
}
@@ -107,24 +116,31 @@ export async function exportIncrementalVm(
})
)
return {
version: '1.1.0',
vbds,
vdis,
vifs,
vm: {
...vm,
return Object.defineProperty(
{
version: '1.1.0',
vbds,
vdis,
vifs,
vm: {
...vm,
},
vtpms,
},
vtpms,
disks,
}
'streams',
{
configurable: true,
value: streams,
writable: true,
}
)
}
export const importIncrementalVm = defer(async function importIncrementalVm(
$defer,
incrementalVm,
sr,
{ cancelToken = CancelToken.none, newMacAddresses = false, targetRef = undefined } = {}
{ cancelToken = CancelToken.none, newMacAddresses = false } = {}
) {
const { version } = incrementalVm
if (compareVersions(version, '1.0.0') < 0) {
@@ -136,105 +152,44 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
const vdiRecords = incrementalVm.vdis
// When targetRef is provided, update the existing VM instead of creating a new one.
const targetVm = targetRef !== undefined ? xapi.getObjectByRef(targetRef, undefined) : undefined
const isUpdate = targetVm !== undefined && !targetVm.is_a_snapshot && !targetVm.is_a_template
// 0. Create suspend_VDI (only when creating a new VM).
// 0. Create suspend_VDI
let suspendVdi
if (!isUpdate) {
if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
const vdi = vdiRecords[vmRecord.suspend_VDI]
if (vdi === undefined) {
Task.warning('Suspend VDI not available for this suspended VM', {
vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
})
} else {
suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
$defer.onFailure(() => suspendVdi.$destroy())
}
if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
const vdi = vdiRecords[vmRecord.suspend_VDI]
if (vdi === undefined) {
Task.warning('Suspend VDI not available for this suspended VM', {
vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
})
} else {
suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
$defer.onFailure(() => suspendVdi.$destroy())
}
}
// 1. Create the VM or update the existing one.
let vmRef
if (isUpdate) {
vmRef = targetRef
await Promise.all([
xapi.setFields('VM', vmRef, {
actions_after_crash: vmRecord.actions_after_crash,
actions_after_reboot: vmRecord.actions_after_reboot,
actions_after_shutdown: vmRecord.actions_after_shutdown,
domain_type: vmRecord.domain_type,
ha_restart_priority: vmRecord.ha_restart_priority,
has_vendor_device: vmRecord.has_vendor_device,
HVM_boot_params: vmRecord.HVM_boot_params,
HVM_boot_policy: vmRecord.HVM_boot_policy,
HVM_shadow_multiplier: vmRecord.HVM_shadow_multiplier,
memory_dynamic_max: vmRecord.memory_dynamic_max,
memory_dynamic_min: vmRecord.memory_dynamic_min,
memory_static_max: vmRecord.memory_static_max,
memory_static_min: vmRecord.memory_static_min,
name_description: vmRecord.name_description,
order: vmRecord.order,
other_config: vmRecord.other_config,
platform: vmRecord.platform,
PV_args: vmRecord.PV_args,
PV_bootloader: vmRecord.PV_bootloader,
PV_bootloader_args: vmRecord.PV_bootloader_args,
PV_kernel: vmRecord.PV_kernel,
PV_legacy_args: vmRecord.PV_legacy_args,
PV_ramdisk: vmRecord.PV_ramdisk,
recommendations: vmRecord.recommendations,
shutdown_delay: vmRecord.shutdown_delay,
start_delay: vmRecord.start_delay,
suspend_SR: vmRecord.suspend_SR,
tags: vmRecord.tags,
user_version: vmRecord.user_version,
VCPUs_at_startup: vmRecord.VCPUs_at_startup,
VCPUs_max: vmRecord.VCPUs_max,
VCPUs_params: vmRecord.VCPUs_params,
xenstore_data: vmRecord.xenstore_data,
}),
targetVm.update_blocked_operations({
// 1. Create the VM.
const vmRef = await xapi.VM_create(
{
...vmRecord,
affinity: undefined,
blocked_operations: {
...vmRecord.blocked_operations,
start: 'Importing…',
start_on: 'Importing…',
}),
])
} else {
vmRef = await xapi.VM_create(
{
...vmRecord,
affinity: undefined,
blocked_operations: {
...vmRecord.blocked_operations,
start: 'Importing…',
start_on: 'Importing…',
},
ha_always_run: false,
is_a_template: false,
name_label: '[Importing…] ' + vmRecord.name_label,
},
{
bios_strings: vmRecord.bios_strings,
generateMacSeed: newMacAddresses,
suspend_VDI: suspendVdi?.$ref,
}
)
$defer.onFailure.call(xapi, 'VM_destroy', vmRef)
}
ha_always_run: false,
is_a_template: false,
name_label: '[Importing…] ' + vmRecord.name_label,
},
{
bios_strings: vmRecord.bios_strings,
generateMacSeed: newMacAddresses,
suspend_VDI: suspendVdi?.$ref,
}
)
$defer.onFailure.call(xapi, 'VM_destroy', vmRef)
// 2. Delete all VBDs which may have been created by the import.
// In update mode, also collect the VDI refs so orphaned VDIs can be destroyed after new ones are created.
const existingVbdRefs = await xapi.getField('VM', vmRef, 'VBDs')
const oldVdiRefs = new Set(
isUpdate
? (await asyncMap(existingVbdRefs, ref => xapi.getField('VBD', ref, 'VDI'))).filter(
ref => ref !== undefined && ref !== 'OpaqueRef:NULL'
)
: []
)
await asyncMap(existingVbdRefs, ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
await asyncMap(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
// 3. Create VDIs & VBDs.
const vbdRecords = incrementalVm.vbds
@@ -244,19 +199,11 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
const vdi = vdiRecords[vdiRef]
let newVdi
if (vdi.baseVdi?.$ref !== undefined) {
if (isUpdate) {
// In update mode, reuse the existing target VDI directly — no clone needed.
newVdi = vdi.baseVdi
oldVdiRefs.delete(newVdi.$ref)
} else {
newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
$defer.onFailure(() => newVdi.$destroy())
}
if (vdi.baseVdi !== undefined) {
newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
$defer.onFailure(() => newVdi.$destroy())
await newVdi.update_other_config(COPY_OF, vdi.uuid)
if (vdi.other_config[CONTENT_KEY] !== undefined) {
await newVdi.update_other_config(CONTENT_KEY, vdi.other_config[CONTENT_KEY])
}
if (vdi.virtual_size > newVdi.virtual_size) {
await newVdi.$callAsync('resize', vdi.virtual_size)
}
@@ -282,16 +229,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
newVdis[vdiRef] = newVdi
})
// 3.5. Destroy old VDIs that are no longer attached to the VM.
// Uses ignoreErrors because some storage backends refuse to destroy a VDI that still has snapshot children;
// those VDIs will become truly orphaned once the old snapshot VMs are cleaned up by _deleteOldEntries.
await asyncMap([...oldVdiRefs], ref => ignoreErrors.call(xapi.call('VDI.destroy', ref)))
// 4. For updates, destroy existing VIFs before recreating them.
if (isUpdate) {
await asyncMap(await xapi.getField('VM', vmRef, 'VIFs'), ref => ignoreErrors.call(xapi.call('VIF.destroy', ref)))
}
const networksByNameLabelByVlan = {}
let defaultNetwork
Object.values(xapi.objects.all).forEach(object => {
@@ -307,28 +244,21 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
}
})
const { disks } = incrementalVm
const { streams } = incrementalVm
await Promise.all([
// Import VDI contents.
cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
for (const disk of ensureArray(disks[id])) {
if (disk === null) {
// we restore a backup and reuse completely a local snapshot
for (let stream of ensureArray(streams[`${id}.vhd`])) {
if (stream === null) {
// we restore a backup and reuse completly a local snapshot
continue
}
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
let stream, format
if (vdi.virtual_size > VHD_MAX_SIZE) {
stream = await toQcow2Stream(disk)
format = 'qcow2'
} else {
stream = await toVhdStream(disk)
format = 'vhd'
if (typeof stream === 'function') {
stream = await stream()
}
await vdi.$importContent(stream, { cancelToken, format })
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
}
}),
@@ -370,16 +300,10 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
await xapi.VTPM_create({ VM: vmRef, contents })
})
)
const vm = await xapi.getRecord('VM', vmRef)
await Promise.all([
vmRecord.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
xapi.setField('VM', vmRef, 'name_label', vmRecord.name_label),
// correctly unlock the VM and reapply the target blocked operations
vm.update_blocked_operations({
start: null,
start_on: null,
...vmRecord.blocked_operations,
}),
incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
xapi.setField('VM', vmRef, 'name_label', incrementalVm.vm.name_label),
])
return vmRef

View File

@@ -44,7 +44,7 @@ const isValidTar = async (handler, size, fd) => {
return buf.every(_ => _ === 0)
}
// TODO: find a heuristic for compressed files
// TODO: find an heuristic for compressed files
export async function isValidXva(path) {
const handler = this._handler

View File

@@ -20,31 +20,13 @@ const IGNORED_PARTITION_TYPES = {
0xd5: true,
0x82: true, // swap
// GPT Linux swap GUID
'0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': true,
}
// MBR LVM type
export const LVM_PARTITION_TYPE_MBR = 0x8e
// GPT LVM type
export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
export const LVM_PARTITION_TYPE = 0x8e
const parsePartxLine = createParser({
keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
valueTransform: (value, key) => {
if (key === 'start' || key === 'size') {
return +value
}
// For GPT partitions, type is a UUID string
if (key === 'type' && !value.startsWith('0x')) {
return value.toLowerCase()
}
// For MBR partitions, type is a hex number as string (e.g., "0x8e")
if (key === 'type' && value.startsWith('0x')) {
return parseInt(value, 16)
}
return value
},
valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
})
// returns an empty array in case of a non-partitioned disk

View File

@@ -28,11 +28,6 @@ export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
// the VM ( not the snapshot) uuid
export const VM_UUID = 'xo:backup:vm'
// in `other_config` of source snapshot VDIs and replicated target VDIs
// contains the UUID of the source snapshot VDI whose content this VDI represents
// allows direction-agnostic discovery of common base VDIs (e.g. for failback/reverse replication)
export const CONTENT_KEY = 'xo:backup:contentKey'
async function listVdiRefs(xapi, vmRef) {
return xapi.VM_getDisks(vmRef)
}
@@ -48,7 +43,7 @@ async function getDeltaChainLength(xapi, type, ref) {
}
/**
* set the delta chain length ( number of delta since last base backup) to a VM and its associated VDIs
* set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
@@ -63,7 +58,7 @@ export async function setVmDeltaChainLength(xapi, vmRef, length) {
/**
* Compute the delta chain length of a VM and its associated VDIs
* if there is a discrepancy, use, the highest value
* if there is a discrependcy, use, the highest value
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
@@ -75,8 +70,7 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
/**
*
* Reset the other_config field related to backups of a VM and its VDIs
*
* Reset the other_config field of a VM and its VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
@@ -96,6 +90,39 @@ export function resetVmOtherConfig(xapi, vmRef) {
})
}
/**
*
* used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
*
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
*/
export async function populateVdisOtherConfig(xapi, vmRef) {
const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
const {
[DATETIME]: datetime,
[DELTA_CHAIN_LENGTH]: chainLength,
[EXPORTED_SUCCESSFULLY]: successfully,
[JOB_ID]: jobId,
[REPLICATED_TO_SR_UUID]: replicatedTo,
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
} = otherConfig
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
xapi.setFieldEntries(type, ref, 'other_config', {
[DATETIME]: datetime,
[DELTA_CHAIN_LENGTH]: chainLength,
[EXPORTED_SUCCESSFULLY]: successfully,
[JOB_ID]: jobId,
[REPLICATED_TO_SR_UUID]: replicatedTo,
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
})
)
}
/**
*
* set the other_config key related to a backup of a VM and its associated VDIs
@@ -125,7 +152,7 @@ export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, schedule
}
/**
*
* mark the export of he VM and its VDIs as successful
* mark the export of he VM and its VDIs as successfull
*
* @param {Xapi} xapi
* @param {String} vmRef
@@ -136,24 +163,3 @@ export async function markExportSuccessfull(xapi, vmRef) {
xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
)
}
/**
* Set CONTENT_KEY on each VDI of a VM snapshot to its own UUID.
*
* This marks each snapshot VDI with a unique content identifier that can be
* propagated to replicated copies for direction-agnostic base VDI discovery
* (e.g. for failback / reverse replication).
*
* @param {Xapi} xapi
* @param {String} snapshotRef
* @returns {Promise}
*/
export async function setVmSnapshotContentKeys(xapi, snapshotRef) {
return applyToVmAndVdis(xapi, snapshotRef, async (type, ref) => {
if (type !== 'VDI') {
return
}
const uuid = await xapi.getField(type, ref, 'uuid')
return xapi.setFieldEntry(type, ref, 'other_config', CONTENT_KEY, uuid)
})
}

View File

@@ -4,12 +4,11 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
import { Throttle } from '@vates/generator-toolbox'
import createStreamThrottle from './_createStreamThrottle.mjs'
const noop = Function.prototype
@@ -42,7 +41,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const schedule = this._schedule
const settings = this._settings
const throttleGenerator = new Throttle(settings.maxExportRate)
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
@@ -91,8 +89,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
schedule,
settings: vmSettings,
sourceRemoteAdapter,
throttleGenerator, // for incrementals
throttleStream, // for full
throttleStream,
vmUuid,
}
let vmBackup

View File

@@ -4,13 +4,12 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { runTask } from './_runTask.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
import { FullXapi } from './_vmRunners/FullXapi.mjs'
import { Throttle } from '@vates/generator-toolbox'
import createStreamThrottle from './_createStreamThrottle.mjs'
const noop = Function.prototype
@@ -56,7 +55,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const schedule = this._schedule
const settings = this._settings
const throttleGenerator = new Throttle(settings.maxExportRate)
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
@@ -149,7 +147,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
schedule,
settings: vmSettings,
srs,
throttleGenerator,
throttleStream,
vm,
}

View File

@@ -2,11 +2,10 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
import { watchStreamSize } from '../../_watchStreamSize.mjs'
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
_getRemoteWriters() {
return [FullRemoteWriter, AggregatedFullRemoteWriter]
_getRemoteWriter() {
return FullRemoteWriter
}
_filterTransferList(transferList) {
@@ -17,7 +16,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
for (const metadata of transferList) {
const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
const sizeContainer = watchStreamSize(stream)
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
@@ -25,9 +24,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
writer =>
writer.run({
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additional properties to it
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
streamLength: stream.length,
maxStreamLength: stream.maxStreamLength, // on encrypted source
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
@@ -38,6 +36,5 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
// for healthcheck
this._tags = metadata.vm.tags
}
this._hasTransferredData = transferList.length > 0
}
}

View File

@@ -5,45 +5,23 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
import { watchStreamSize } from '../../_watchStreamSize.mjs'
import { AbstractXapi } from './_AbstractXapi.mjs'
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
import { AggregatedFullXapiWriter } from '../_writers/AggregatedFullXapiWriter.mjs'
const { debug } = createLogger('xo:backups:FullXapiVmBackup')
export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
_getWriters() {
return [FullRemoteWriter, FullXapiWriter, AggregatedFullRemoteWriter, AggregatedFullXapiWriter]
return [FullRemoteWriter, FullXapiWriter]
}
async _mustDoSnapshot() {
_mustDoSnapshot() {
const vm = this._vm
const settings = this._settings
// General setting forcing snapshot at all times
if (settings.unconditionalSnapshot) {
return true
}
// If offline backup option is not checked
// and vm is running, we make a snapshot
if (!settings.offlineBackup && vm.power_state === 'Running') {
return true
}
// General setting used for snapshot retention
// see schedule in backupJob edition
if (settings.snapshotRetention !== 0) {
return true
}
const vdis = await this._xapi.getRecords('VDI', await vm.$getDisks())
const hasNOBAKtag = vdis.some(idx => idx.name_label.includes('[NOBAK]'))
// if there is a disk with NOBAK tag
// we need to make a snapshot
if (hasNOBAKtag) {
return true
}
return (
settings.unconditionalSnapshot ||
(!settings.offlineBackup && vm.power_state === 'Running') ||
settings.snapshotRetention !== 0
)
}
_selectBaseVm() {}
@@ -65,17 +43,10 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
for (const vdiRef of vdis) {
const vdi = await this._xapi.getRecord('VDI', vdiRef)
// the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
// of the real stream size in general, since a disk is rarely completely full
// the size a of fully allocated vdi will be virtual_size exaclty, it's a gross over evaluation
// of the real stream size in general, since a disk is never completly full
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
// xva files are tar archive with data cut in 1MB blocks
// each block is signed with a hash (in xxhash format)
// each of these blocks + hash add one tar header entry in the file
const XVA_BLOCK_SIZE = 1024 * 1024
const nbBlocks = Math.ceil(vdi.virtual_size / XVA_BLOCK_SIZE)
const headersSize = nbBlocks * (512 /* file header */ + 512 /* xxhash header */ + 512) /* xhxhash size */
maxStreamLength += XVA_BLOCK_SIZE * nbBlocks + headersSize
maxStreamLength += vdi.virtual_size
}
const sizeContainer = watchStreamSize(stream)

Some files were not shown because too many files have changed in this diff Show More