Compare commits

..

5 Commits

Author SHA1 Message Date
Sebastian CZECH
d62433148b update 2025-05-07 16:08:49 +02:00
Sebastian CZECH
d44364d4d5 update 2025-05-07 14:08:15 +02:00
Sebastian CZECH
9118dc0835 update 2025-05-07 14:05:35 +02:00
Sebastian CZECH
97bf035a9f update 2025-05-07 14:03:28 +02:00
Sebastian CZECH
1e7ac62c51 feat(web-core): create task item component 2025-04-30 03:17:11 +02:00
1083 changed files with 11304 additions and 35048 deletions

View File

@@ -171,6 +171,7 @@ module.exports = {
'vue/no-empty-component-block': 'error',
'vue/no-multiple-objects-in-class': 'error',
'vue/no-ref-object-reactivity-loss': 'error',
'vue/no-required-prop-with-default': 'error',
'vue/no-static-inline-styles': 'error',
'vue/no-template-target-blank': 'error',
'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
@@ -216,16 +217,7 @@ module.exports = {
'@intlify/vue-i18n/no-raw-text': [
'error',
{
ignoreText: [
'Xen Orchestra Lite',
'Xen Orchestra',
'Xen Orchestra Appliance',
'XO Lite',
'XCP-ng',
'XO 5',
'404',
'⚠️',
],
ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
},
],
'@intlify/vue-i18n/no-v-html': 'error',

View File

@@ -40,8 +40,8 @@ body:
- type: markdown
attributes:
value: |
- Don't forget to [read this first](https://docs.xen-orchestra.com/community)
- As well as follow [this guide](https://docs.xen-orchestra.com/community#report-a-bug)
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
- type: input
id: xo-sources-commit-number
attributes:

View File

@@ -1,4 +1,4 @@
name: Continuous Integration
name: Continous Integration
on: push
jobs:
CI:

View File

@@ -16,7 +16,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
- `signal`: an abort signal to stop the iteration
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'

View File

@@ -34,7 +34,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
- `signal`: an abort signal to stop the iteration
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'

View File

@@ -7,7 +7,7 @@
* @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
* @param {T} data2 - The second data array or buffer to compare.
* @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided; otherwise, undefined.
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
*/
module.exports = function diff(data1, data2, cb) {
let result

View File

@@ -14,7 +14,7 @@ import { deduped } from '@vates/disposable/deduped'
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
const getConnection = deduped(async function (host)) {
const connection = new Connection(host)
return new Disposable(connection, () => connection.close())
return new Disposabe(connection, () => connection.close())
}, host => [host])
```

View File

@@ -32,7 +32,7 @@ import { deduped } from '@vates/disposable/deduped'
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
const getConnection = deduped(async function (host)) {
const connection = new Connection(host)
return new Disposable(connection, () => connection.close())
return new Disposabe(connection, () => connection.close())
}, host => [host])
```

View File

@@ -28,7 +28,7 @@ fs.stat('autoexec.bat', function (e, stats) {
## API
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back an instance of an [`fs`](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back the API documented [here](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
- `'ready'` event — fired on `fs` when initial volume information has been determined and the API is ready to use. It is safe to call other `fs` methods before this fires **only if** you are sure the first sector will be readable and represents a valid FAT volume.
- `'error'` event — fired if initialization fails for whatever reason.
@@ -36,7 +36,7 @@ fs.stat('autoexec.bat', function (e, stats) {
The `opts` dictionary you pass to `fatfs.createFileSystem` can contain any of the following options:
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overridden to `true`.
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overriden to `true`.
- `noatime` — The FAT filesystem can track the last access time (just a date, actually) but this means every read operation would also incur some write overhead. Defaults to `true`, meaning by default access times will **not** be updated on reads. Set this to `false` to track access times.
- `modmode` — chooses how `fs.chmod` (and the mode field from `fs.stat`–family calls) should map FAT attributes to POSIX permissions. Set to the number `0111` to map the readonly flag to the user's write bit being unset, and the archive/system/hidden flags to the user/group/other executable bits respectively. Set to the number `07000` to map the readonly flag to _all_ write bits being unset, and the archive/system/hidden flags to the sticky/setgid/setuid bits respectively. Set to `null` for readonly mapping. Defaults to `0111`.
- `umask` — any bits _set_ in this octal number will be _unset_ in the 'mode' field from `fs.stat`–family calls. It does not affect anything else. Defaults to `process.umask()`, or `0022` if that is unavailable.
@@ -53,7 +53,7 @@ Well, sort of…
### Temporary
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please, please, please, **make sure you have a backup** of any important drive/image/card you unleash this upon.
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please please please **make sure you have a backup** of any important drive/image/card you unleash this upon.
- A few methods are not quite implemented, either: `fs.rename`, `fs.unlink` and `fs.rmdir`, as well as `fs.watchFile`/`fs.unwatchFile` and `fs.watch`. These are Coming Soon™.
- There are several internal housekeeping items (redundant FAT tables, extra FAT32 information, etc.) that are not done. These do not seem to affect interop, but you may see warnings when repairing a filesystem written by this module.
- Oh, and not to scare you, but if an IO error happens while writing, the library usually just bails — bubbling an error up to your callback as if it were a hot potato. Although some attempt has been made to do separate writes in the safest order (e.g. allocating an additional file cluster, then appending data into it, and then finally updating the file's size), but this behavior has not been thoroughly audited for all operations. There's certainly no attempt to retry/cleanup/rollback if a multi-step change runs into trouble partway through.

View File

@@ -29,7 +29,7 @@ The `remoteUrl` can be found using the following command from the XOA CLI, and t
`xo-server-db ls remote`
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
The `vhdPathInRemote` follows a file structure described [here](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
## Restore a file from a VHD using `fuse-vhd` CLI
@@ -42,7 +42,7 @@ The `vhdPathInRemote` follows a file structure described in [VM backups](https:/
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
4. Mount the disk. Depending on the partition type, you may need to add some additionnal options. The `norecovery` option is used for ext3/ext4/xfs file systems, otherwise remove this option.
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`

View File

@@ -45,7 +45,7 @@ The `remoteUrl` can be found using the following command from the XOA CLI, and t
`xo-server-db ls remote`
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
The `vhdPathInRemote` follows a file structure described [here](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
## Restore a file from a VHD using `fuse-vhd` CLI
@@ -58,7 +58,7 @@ The `vhdPathInRemote` follows a file structure described in [VM backups](https:/
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
4. Mount the disk. Depending on the partition type, you may need to add some additionnal options. The `norecovery` option is used for ext3/ext4/xfs file systems, otherwise remove this option.
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`

View File

@@ -19,12 +19,12 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.6.2",
"@xen-orchestra/fs": "^4.5.0",
"@xen-orchestra/log": "^0.7.1",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.14.0"
"vhd-lib": "^4.11.3"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"

View File

@@ -30,14 +30,14 @@ const wrappedGenerator = new Throttle(sourceGenerator, speed)
### Synchronized
Fork a generator. The rules are:
Fork a generator. The rules ares:
- if the source returns, all the forks return
- if the forks error, all the forks error with the same error
- if a fork returns, it is stopped, but the generator continue with the other
- if a fork errors, it is stopped, but the generator continue with the other
- if all the forks return, the source is stopped
- if all the forks error, the source is errored with the last error
- if the source returns, all the forks returns
- if the forks errors, all the forks errors with the same error
- if a the fork return , it is stopped, but the generator continue with the other
- if a the fork error , it is stopped, but the generator continue with the other
- if all the fork return , the source is stopped
- if all the fork error , the source is errored with the last error
- the source start producing a packet when the fastest forked ask for it
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory

View File

@@ -46,14 +46,14 @@ const wrappedGenerator = new Throttle(sourceGenerator, speed)
### Synchronized
Fork a generator. The rules are:
Fork a generator. The rules ares:
- if the source returns, all the forks return
- if the forks error, all the forks error with the same error
- if a fork returns, it is stopped, but the generator continue with the other
- if a fork errors, it is stopped, but the generator continue with the other
- if all the forks return, the source is stopped
- if all the forks error, the source is errored with the last error
- if the source returns, all the forks returns
- if the forks errors, all the forks errors with the same error
- if a the fork return , it is stopped, but the generator continue with the other
- if a the fork error , it is stopped, but the generator continue with the other
- if all the fork return , the source is stopped
- if all the fork error , the source is errored with the last error
- the source start producing a packet when the fastest forked ask for it
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory

View File

@@ -1,13 +1,12 @@
{
"name": "@vates/generator-toolbox",
"version": "1.0.4",
"version": "1.0.1",
"main": "dist/index.mjs",
"license": "MIT",
"private": false,
"type": "module",
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^5.0.1",
"@types/node": "^20.6",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0"
@@ -16,10 +15,7 @@
"build": "tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"prebuild": "rimraf dist/",
"predev": "yarn run prebuild",
"test": "tsc && node --test **/*.test.mjs",
"clean": "rimraf dist/"
"test": "tsc && node --test **/*.test.mjs"
},
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/generator-toolbox",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",

View File

@@ -17,10 +17,8 @@ export class Synchronized<T, TReturn, TNext> {
this.#source = source
}
fork(uid: string): AsyncGenerator<T, TReturn, TNext> {
fork(uid: string): AsyncGenerator {
assert.strictEqual(this.#started, false, `can't create a fork after consuming the data`)
assert.strictEqual(this.#forks.has(uid), false, `a fork with id ${uid} already exists`)
const fork = new Forked<T, TReturn, TNext>(this, uid)
this.#forks.set(uid, fork)
return fork
@@ -85,7 +83,7 @@ export class Synchronized<T, TReturn, TNext> {
// already removed
return { done: true, value: undefined }
}
throw new Error(`trying to remove fork with uid ${uid} that is not a fork of this one`)
throw new Error(`trying to remove fork wih uid ${uid} that is not a fork of this one`)
}
this.#forks.delete(uid)
this.#waitingForks.delete(uid)
@@ -144,14 +142,7 @@ class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
return this.#parent.remove(this.#uid, e)
}
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
while (true) {
const res = await this.next()
if (res.done) {
break
}
yield res.value
}
[Symbol.asyncIterator](): AsyncGenerator<T> {
return this
}
}

View File

@@ -139,12 +139,4 @@ suite('error', () => {
await first.next()
assert.throws(() => forker.fork('second'))
})
test('It should not allow fork with the same uid', async () => {
const progress = { yielded: 0 }
const generator = makeRangeGenerator(5, progress)
const forker = new Synchronized(generator)
forker.fork('first')
forker.fork('second')
assert.throws(() => forker.fork('first'))
})
})

View File

@@ -1,4 +1,4 @@
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.

View File

@@ -16,7 +16,7 @@ npm install --save @vates/json-hash
## Usage
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.

View File

@@ -1,6 +1,6 @@
export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negotiation
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
export const NBD_OPT_EXPORT_NAME = 1
export const NBD_OPT_ABORT = 2
export const NBD_OPT_LIST = 3

View File

@@ -1,7 +1,7 @@
import assert from 'node:assert'
import { Socket } from 'node:net'
import { connect } from 'node:tls'
import { pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
import { readChunkStrict } from '@vates/read-chunk'
import { createLogger } from '@xen-orchestra/log'
import {
@@ -20,7 +20,6 @@ import {
OPTS_MAGIC,
NBD_CMD_DISC,
} from './constants.mjs'
const { warn } = createLogger('vates:nbd-client')
// documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -43,9 +42,9 @@ export default class NbdClient {
// AFAIK, there is no guaranty the server answers in the same order as the queries
// so we handle a backlog of command waiting for response and handle concurrency manually
#waitingForResponse // there is already a listener waiting for a response
#waitingForResponse // there is already a listenner waiting for a response
#nextCommandQueryId = BigInt(0)
#commandQueryBacklog // map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
#commandQueryBacklog // map of command waiting for an response queryId => { size/*in byte*/, resolve, reject}
#connected = false
#reconnectingPromise
@@ -196,7 +195,7 @@ export default class NbdClient {
}
// send export name we want to access.
// it's implicitly closing the negotiation phase.
// it's implictly closing the negociation phase.
await this.#write(OPTS_MAGIC)
await this.#writeInt32(NBD_OPT_EXPORT_NAME)
const exportNameBuffer = Buffer.from(this.#exportName)
@@ -219,22 +218,8 @@ export default class NbdClient {
}
#write(buffer) {
let timeout
const messageTimeout = this.#messageTimeout
const socket = this.#serverSocket
return Promise.race([
new Promise((resolve, reject) => {
timeout = setTimeout(() => {
reject(new Error('timeout'))
}, messageTimeout)
}),
new Promise(resolve =>
socket.write(buffer, () => {
clearTimeout(timeout)
resolve()
})
),
])
const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
return pTimeout.call(promise, this.#messageTimeout)
}
async #readInt32() {

View File

@@ -30,8 +30,6 @@ export default class MultiNbdClient {
* open nbdConcurrency connections to NBD servers
* it must obtain at least one connection to succeed
* it tries to spread connections on multiple host
*
* @returns {Promise<void>}
*/
async connect() {
const candidates = [...this.#settings]
@@ -70,9 +68,7 @@ export default class MultiNbdClient {
await _connect()
}
if (this.#clients.length === 0) {
const error = new Error(`Fail to connect to any Nbd client`, { nbdInfos: this.#settings })
error.code = 'NO_NBD_AVAILABLE'
throw error
throw new Error(`Fail to connect to any Nbd client`)
}
if (this.#clients.length < this.#nbdConcurrency) {
warn(
@@ -81,30 +77,17 @@ export default class MultiNbdClient {
}
}
/**
* @returns {Promise<void>}
*/
async disconnect() {
await asyncEach(this.#clients, client => client.disconnect(), {
stopOnError: false,
})
}
/**
*
* @param {number} index
* @param {number} size
* @returns {Promise<Buffer>}
*/
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
const clientId = index % this.#clients.length
return this.#clients[clientId].readBlock(index, size)
}
/**
*
* @param {AsyncGenerator<Buffer>} indexGenerator
*/
async *readBlocks(indexGenerator) {
// default : read all blocks
const readAhead = []

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.1.4",
"version": "3.1.2",
"engines": {
"node": ">=14.0"
},
@@ -24,7 +24,7 @@
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.7.1",
"promise-toolbox": "^0.21.0",
"xen-api": "^4.7.3"
"xen-api": "^4.7.1"
},
"devDependencies": {
"tap": "^18.7.0",

View File

@@ -24,7 +24,7 @@ $ npm install node-vsphere-soap --save
## Sample Code
### To connect to a vCenter server
### To connect to a vCenter server:
var nvs = require('node-vsphere-soap');
var vc = new nvs.Client(host, user, password, sslVerify);
@@ -54,7 +54,7 @@ $ npm install node-vsphere-soap --save
- userName - username of authenticated user
- fullName - full name of authenticated user
### To run a command
### To run a command:
var vcCmd = vc.runCommand( commandToRun, arguments );
vcCmd.once('result', function( result, raw, soapHeader) {

View File

@@ -7,7 +7,7 @@
This file creates the Client class
- when the class is instantiated, a connection will be made to the ESXi/vCenter server to verify that the creds are good
- upon a bad login, the connection will be terminated
- upon a bad login, the connnection will be terminated
*/

View File

@@ -1,6 +1,6 @@
### Usual workflow
> This section presents how this library should be used to implement a classic two factor authentication.
> This section presents how this library should be used to implement a classic two factor authentification.
#### Setup

View File

@@ -18,7 +18,7 @@ npm install --save @vates/otp
### Usual workflow
> This section presents how this library should be used to implement a classic two factor authentication.
> This section presents how this library should be used to implement a classic two factor authentification.
#### Setup

View File

@@ -63,16 +63,16 @@ task.failure(error)
Inside a task:
```js
// the abort signal of the current task if any; otherwise, is `undefined`
// the abort signal of the current task if any, otherwise is `undefined`
Task.abortSignal
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.info(message, data)
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.warning(message, data)
// attaches a property to the current task if any; otherwise, does nothing
// attaches a property to the current task if any, otherwise does nothing
//
// the latest value takes precedence
//
@@ -149,7 +149,7 @@ const onProgress = makeOnProgress({
// This function is called each time a root task or a subtask is updated.
//
// `taskLog.$root` can be used to unconditionally access the root task.
// `taskLog.$root` can be used to uncondionally access the root task.
onTaskUpdate(taskLog) {},
})

View File

@@ -79,16 +79,16 @@ task.failure(error)
Inside a task:
```js
// the abort signal of the current task if any; otherwise, is `undefined`
// the abort signal of the current task if any, otherwise is `undefined`
Task.abortSignal
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.info(message, data)
// sends an info on the current task if any; otherwise, does nothing
// sends an info on the current task if any, otherwise does nothing
Task.warning(message, data)
// attaches a property to the current task if any; otherwise, does nothing
// attaches a property to the current task if any, otherwise does nothing
//
// the latest value takes precedence
//
@@ -165,7 +165,7 @@ const onProgress = makeOnProgress({
// This function is called each time a root task or a subtask is updated.
//
// `taskLog.$root` can be used to unconditionally access the root task.
// `taskLog.$root` can be used to uncondionally access the root task.
onTaskUpdate(taskLog) {},
})

View File

@@ -29,7 +29,7 @@ function createTask(opts) {
}
describe('Task', function () {
describe('constructor', function () {
describe('contructor', function () {
it('data properties are passed to the start event', async function () {
const properties = { foo: 0, bar: 1 }
const task = createTask({ properties })
@@ -265,7 +265,7 @@ describe('Task', function () {
assert.throws(() => task.set(name, value), { message: /^task has not started yet\b/ })
})
it(`emits a property message`, async function () {
it(`emits an property message`, async function () {
const task = createTask()
await task.run(async () => {
await Task.run(() => {
@@ -339,7 +339,7 @@ describe('Task', function () {
Task.warning('foo')
})
it('emits a warning message when run inside a task', async function () {
it('emits an warning message when run inside a task', async function () {
const task = createTask()
await task.run(() => {
Task.warning('foo')

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.6.2",
"version": "0.6.1",
"engines": {
"node": ">=14"
},

View File

@@ -5,8 +5,4 @@ import tseslint from 'typescript-eslint'
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,7 +1,7 @@
{
"name": "@vates/types",
"private": false,
"version": "1.10.0",
"version": "1.2.0",
"main": "./dist/index.mjs",
"exports": {
".": {
@@ -17,10 +17,6 @@
"./lib/vates/*": {
"default": "./dist/lib/vates-*.mjs",
"type": "./dist/lib/vates-*.d.mts"
},
"./lib/xen-orchestra/*": {
"default": "./dist/lib/xen-orchestra-*.mjs",
"type": "./dist/lib/xen-orchestra-*.d.mts"
}
},
"type": "module",

View File

@@ -1,5 +1,3 @@
import type { XoHost } from './xo.mjs'
declare const __brand: unique symbol
export type Branded<TBrand extends string, TType = string> = TType & { [__brand]: TBrand }
@@ -651,10 +649,6 @@ export const OPAQUE_REF = { EMPTY: 'OpaqueRef:NULL' } as const
export type OPAQUE_REF_NULL = (typeof OPAQUE_REF)['EMPTY']
export const BACKUP_TYPE = { backup: 'backup', metadata: 'metadataBackup', mirror: 'mirrorBackup' } as const
export type BACKUP_TYPE = (typeof BACKUP_TYPE)[keyof typeof BACKUP_TYPE]
// ----- XAPI Stats
type XapiStatsResponse<T> = {
@@ -662,73 +656,47 @@ type XapiStatsResponse<T> = {
interval: number
stats: T
}
type StatValues = (number | null)[]
type RecordStatValues = Record<string, StatValues>
export type XapiStatsGranularity = 'seconds' | 'minutes' | 'hours' | 'days'
export type XapiHostStatsRaw = {
cpus?: RecordStatValues
ioThroughput?: {
r: RecordStatValues
w: RecordStatValues
export type XapiHostStats = XapiStatsResponse<{
cpus: Record<string, number[]>
ioThroughput: {
r: Record<string, number[]>
w: Record<string, number[]>
}
iops?: {
r: RecordStatValues
w: RecordStatValues
iops: {
r: Record<string, number[]>
w: Record<string, number[]>
}
iowait?: RecordStatValues
latency?: {
r: RecordStatValues
w: RecordStatValues
iowait: Record<string, number[]>
latency: {
r: Record<string, number[]>
w: Record<string, number[]>
}
load?: StatValues
memory?: StatValues
memoryFree?: StatValues
pifs?: {
rx: RecordStatValues
tx: RecordStatValues
load: number[]
memory: number[]
memoryFree: number[]
pifs: {
rx: Record<string, number[]>
tx: Record<string, number[]>
}
}
export type XapiHostStats = XapiStatsResponse<XapiHostStatsRaw>
}>
export type XapiVmStatsRaw = {
cpus?: RecordStatValues
cpuUsage?: StatValues
runstateFullrun?: StatValues
runstateFullContention?: StatValues
runstatePartialRun?: StatValues
runstatePartialContention?: StatValues
runstateConcurrencyHazard?: StatValues
runstateBlocked?: StatValues
iops?: {
r: RecordStatValues
w: RecordStatValues
export type XapiVmStats = XapiStatsResponse<{
cpus: Record<string, number[]>
iops: {
r: Record<string, number[]>
w: Record<string, number[]>
}
memory?: StatValues
memoryFree?: StatValues
memoryTarget?: StatValues
vifs?: {
rx: RecordStatValues
tx: RecordStatValues
memory: number[]
memoryFree?: number[]
vifs: {
rx: Record<string, number[]>
tx: Record<string, number[]>
}
vifErrors?: {
rx: RecordStatValues
tx: RecordStatValues
xvds: {
w: Record<string, number[]>
r: Record<string, number[]>
}
xvds?: {
w?: RecordStatValues
r?: RecordStatValues
total?: RecordStatValues
}
vbdLatency?: {
w: RecordStatValues
r: RecordStatValues
}
vbdIowait?: RecordStatValues
vbdInflight?: RecordStatValues
vbdAvgquSz?: RecordStatValues
}
export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>
}>

View File

@@ -1,4 +1,3 @@
export * from './common.mjs'
export * from './xen-api.mjs'
export * from './xo.mjs'
export * from './lib/xen-orchestra-xapi.mjs'

View File

@@ -1,176 +0,0 @@
import {
WrappedXenApiRecord,
XenApiHost,
XenApiNetworkWrapped,
XenApiRecord,
XenApiSr,
XenApiTask,
XenApiVdi,
XenApiVm,
XenApiVmWrapped,
} from '../xen-api.mjs'
import type { Readable } from 'node:stream'
import type {
XoGpuGroup,
XoVgpuType,
XoHost,
XoNetwork,
XoPif,
XoSr,
XoUser,
XoVdi,
XoVm,
XoVmTemplate,
} from '../xo.mjs'
export type XcpPatches = {
changelog?: {
author: string
date: number
description: string
}
description: string
license: string
name: string
release: string
size: number
url: string
version: string
}
export type XsPatches = {
conflicts?: string[]
date: string
description: string
documentationUrl?: string
guidances: string
name: string
id?: string
paid?: boolean
requirements?: string[]
upgrade?: boolean
url?: string
uuid?: string
}
export interface Xapi {
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
getField<T extends XenApiRecord, K extends keyof T>(
type: Extract<WrappedXenApiRecord, T>['$type'],
ref: T['$ref'],
field: K
): Promise<T[K]>
createNetwork(
params:
| {
name: string
description?: string
mtu?: number
}
| {
name: string
description?: string
pifId: XoPif['id']
mtu?: number
/* between 0 and 4094 */
vlan: number
}
): Promise<XenApiNetworkWrapped>
deleteNetwork(id: XoNetwork['id']): Promise<void>
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
pool_emergencyShutdown(): Promise<void>
resumeVm(id: XoVm['id']): Promise<void>
unpauseVm(id: XoVm['id']): Promise<void>
startVm(
id: XoVm['id'],
opts?: {
bypassMacAddressesCheck?: boolean
force?: boolean
hostId?: XoHost['id']
/**
* if startOnly is true and the VM is not halted, throw VM_BAD_POWER_STATE
* otherwise, unpause/resume the VM
*
* @default false
*/
startOnly?: boolean
}
): Promise<void>
VM_import(
stream: Readable,
srRef?: XenApiSr['$ref'],
onVmCreation?: null | ((vm: XenApiVm) => unknown)
): Promise<XenApiVm['$ref']>
createVm(
templateUuid: XoVmTemplate['uuid'],
metadataVm: {
affinityHost?: XoHost['id']
autoPoweron?: boolean
memory?: number
name_description?: string
name_label: string
nameLabel?: string
clone?: boolean
installRepository?: XoVdi['id'] | '' | null
vdis?: (
| /** Create VDI */ {
name_label: string
size: number
sr?: XoSr['id']
name_description?: string
}
| /** Update VDI*/ {
userdevice: string
name_label?: string
size?: number
sr?: XoSr['id']
name_description?: string
}
| /** Destroy VDI*/ { destroy: true; userdevice: string }
)[]
vifs?: (
| /** Create/update VIF */ {
device?: string
ipv4_allowed?: string[]
ipv6_allowed?: string[]
mac?: string
mtu?: number
network: string
}
| /** Destroy VIF */ { destroy: true; device: string }
)[]
existingVdis?: {
$SR: XoSr['id']
size: number
userdevice: string
}[]
vgpuType?: XoVgpuType['id']
gpuGroup?: XoGpuGroup['id']
copyHostBiosStrings?: boolean
},
checkLimits?: boolean,
creatorId?: XoUser['id'],
opts?: { destroyAllVifs: boolean }
): Promise<XenApiVmWrapped>
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
VM_createCloudInitConfig(
vmRef: XenApiVm['$ref'],
cloudConfig: string,
opts?: { networkConfig?: string }
): Promise<XoVdi['uuid']>
VM_destroy(
vmRef: XenApiVm['$ref'],
opts?: {
deleteDisks?: boolean
force?: boolean
bypassBlockedOperation?: boolean
forceDeleteDefaultTemplate?: boolean
}
): Promise<void>
getResource(
pathname: string,
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
): Promise<{ body: Readable }>
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
}

View File

@@ -64,7 +64,6 @@ import type {
VTPM_OPERATIONS,
VUSB_OPERATIONS,
} from './common.mjs'
import type { Xapi } from './lib/xen-orchestra-xapi.mjs'
// types automatically generated by the @ByScripts script.
// https://github.com/vatesfr/xen-orchestra/tree/wip-xapi-types-generator/%40xen-orchestra/xapi-generator
@@ -78,13 +77,10 @@ type WrapperXenApi<T, Type extends string, Fn = { (): void }> = T & {
$call: Fn
$callAsync: Fn
$type: Type
$snapshot(params: {
cancelToken?: unknown
ignoredVdisTag?: string
name_label?: string
unplugVusbs?: boolean
}): Promise<XenApiVm['$ref']>
$xapi: Xapi
$xapi: {
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
}
}
export interface XenApiSession {
@@ -258,14 +254,6 @@ export interface XenApiPoolUpdate {
type XenApiVmCallMethods = {
(method: 'start', start_paused: boolean, force: boolean): Promise<void>
(method: 'clean_shutdown'): Promise<void>
(method: 'hard_shutdown'): Promise<void>
(method: 'clean_reboot'): Promise<void>
(method: 'hard_reboot'): Promise<void>
(method: 'pause'): Promise<void>
(method: 'suspend'): Promise<void>
(method: 'record_data_source', dataSource: string): Promise<void>
(method: 'forget_data_source_archives', dataSource: string): Promise<void>
}
export interface XenApiVm {
$ref: Branded<'VM'>
@@ -633,7 +621,6 @@ export interface XenApiNetwork {
uuid: string
VIFs: XenApiVif['$ref'][]
}
export interface XenApiNetworkWrapped extends WrapperXenApi<XenApiNetwork, 'network'> {}
export interface XenApiVif {
$ref: Branded<'VIF'>
@@ -718,7 +705,6 @@ export interface XenApiPif {
VLAN_slave_of: XenApiVlan['$ref'][]
VLAN: number
}
export type XenApiPifWrapped = WrapperXenApi<XenApiPif, 'PIF'>
export interface XenApiPifMetrics {
$ref: Branded<'PIF_metrics'>
@@ -774,13 +760,11 @@ export interface XenApiSm {
other_config: Record<string, string>
required_api_version: string
required_cluster_stack: string[]
supported_image_formats?: string[]
type: string
uuid: string
vendor: string
version: string
}
export type XenApiSmWrapped = WrapperXenApi<XenApiSm, 'SM'>
export interface XenApiSr {
$ref: Branded<'SR'>
@@ -893,7 +877,7 @@ export interface XenApiVbd {
unpluggable: boolean
userdevice: string
uuid: string
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
VDI: XenApiVdi['$ref']
VM: XenApiVm['$ref']
}
export type XenApiVbdWrapped = WrapperXenApi<XenApiVbd, 'VBD'>
@@ -1036,7 +1020,6 @@ export interface XenApiPci {
uuid: string
vendor_name: string
}
export type XenApiPciWrapped = WrapperXenApi<XenApiPci, 'PCI'>
export interface XenApiPgpu {
$ref: Branded<'PGPU'>
@@ -1053,7 +1036,6 @@ export interface XenApiPgpu {
supported_VGPU_types: XenApiVgpuType['$ref'][]
uuid: string
}
export type XenApiPgpuWrapped = WrapperXenApi<XenApiPgpu, 'PGPU'>
export interface XenApiGpuGroup {
$ref: Branded<'GPU_group'>
@@ -1068,7 +1050,6 @@ export interface XenApiGpuGroup {
uuid: string
VGPUs: XenApiVgpu['$ref'][]
}
export type XenApiGpuGroupWrapped = WrapperXenApi<XenApiGpuGroup, 'gpuGroup'>
export interface XenApiVgpu {
$ref: Branded<'VGPU'>
@@ -1106,7 +1087,6 @@ export interface XenApiVgpuType {
vendor_name: string
VGPUs: XenApiVgpu['$ref'][]
}
export type XenApiVgpuTypeWrapped = WrapperXenApi<XenApiVgpuType, 'vgpuType'>
export interface XenApiPvsSite {
$ref: Branded<'PVS_site'>
@@ -1356,8 +1336,6 @@ export type XenApiRecord =
export type WrappedXenApiRecord =
| XenApiHostWrapped
| XenApiNetworkWrapped
| XenApiPifWrapped
| XenApiPoolWrapped
| XenApiSrWrapped
| XenApiVbdWrapped
@@ -1366,7 +1344,3 @@ export type WrappedXenApiRecord =
| XenApiVifWrapped
| XenApiVmWrapped
| XenApiVtpmWrapped
| XenApiPciWrapped
| XenApiGpuGroupWrapped
| XenApiPgpuWrapped
| XenApiVgpuTypeWrapped

View File

@@ -1,17 +1,12 @@
// Types based on xapi-object-to-xo
import type {
BACKUP_TYPE,
Branded,
DOMAIN_TYPE,
HOST_ALLOWED_OPERATIONS,
HOST_POWER_STATE,
IP_CONFIGURATION_MODE,
IPV6_CONFIGURATION_MODE,
NETWORK_OPERATIONS,
PGPU_DOM0_ACCESS,
POOL_ALLOWED_OPERATIONS,
PRIMARY_ADDRESS_TYPE,
STORAGE_OPERATIONS,
VDI_OPERATIONS,
VDI_TYPE,
@@ -75,7 +70,7 @@ type BaseXoVm = BaseXapiXo & {
isNestedVirtEnabled: boolean
hasVendorDevice: boolean
high_availability: string
installTime?: number | null
installTime?: number
isFirmwareSupported: boolean
memory: {
dynamic: number[]
@@ -101,7 +96,7 @@ type BaseXoVm = BaseXapiXo & {
secureBoot: boolean
snapshots: XoVmSnapshot['id'][]
startDelay: number
startTime?: number | null
startTime?: number
suspendSr?: XoSr['id']
tags: string[]
vga?: string
@@ -123,7 +118,7 @@ type BaseXoVm = BaseXapiXo & {
export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
body: {
value?: string
value: string
name: string
}
object: {
@@ -133,22 +128,6 @@ export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
}
}
export type XoBackupRepository = {
benchmarks?: { readRate: number; timestamp: number; writeRate: number }[]
enabled: boolean
error?: Record<string, unknown>
id: Branded<'backup-repository'>
name: string
options?: string
proxy?: XoProxy['id']
url: string
}
export type XoGpuGroup = BaseXapiXo & {
id: Branded<'gpu-group'>
type: 'gpuGroup'
}
export type XoGroup = {
id: Branded<'group'>
name: string
@@ -282,65 +261,18 @@ export type XoPbd = BaseXapiXo & {
}
export type XoPci = BaseXapiXo & {
$host?: XoHost['id']
class_name: string
device_name: string
id: Branded<'PCI'>
pci_id: string
type: 'PCI'
}
export type XoPgpu = BaseXapiXo & {
$host: XoHost['id']
$vgpus: XoVgpu['id'][]
dom0Access: PGPU_DOM0_ACCESS
enabledVgpuTypes: XoVgpuType['id'][]
gpuGroup?: XoGpuGroup['id']
host: XoHost['id']
id: Branded<'PGPU'>
isSystemDisplayDevice: boolean
pci?: XoPci['id']
// it seems that supportedVgpuMaxCapcities always return undefined (See: xapi-objet-to-xo)
/**
* @deprecated
*/
supportedVgpuMaxCapcities?: never
supportedVgpuTypes: XoVgpuType['id'][]
type: 'PGPU'
vgpus: XoVgpu['id'][]
}
export type XoPif = BaseXapiXo & {
$host: XoHost['id']
$network: XoNetwork['id']
attached: boolean
bondMaster?: XoPif['id']
bondSalves?: XoPif['id'][]
carrier: boolean
device: string
deviceName?: string
disallowUnplug: boolean
dns: string
gateway: string
id: Branded<'PIF'>
ip: string
ipv6: string[]
ipv6Mode: IPV6_CONFIGURATION_MODE
isBondMaster: boolean
isBondSlave: boolean
mac: string
management: boolean
mode: IP_CONFIGURATION_MODE
mtu: number
netmask: string
physical: boolean
primaryAddressType: PRIMARY_ADDRESS_TYPE
speed?: number
type: 'PIF'
vlan: number
}
export type XoPool = BaseXapiXo & {
@@ -369,65 +301,15 @@ export type XoPool = BaseXapiXo & {
zstdSupported: boolean
}
export type XoProxy = {
id: Branded<'proxy'>
}
type BaseXoJob = {
export type XoJob = {
id: Branded<'job'>
}
// @TODO: create type for complex matcher
export type XoBackupJob = BaseXoJob & {
compression?: 'native' | 'zstd' | ''
proxy?: XoProxy['id']
mode: 'full' | 'delta'
name?: string
remotes?: {
id: XoBackupRepository['id'] | { __or: XoBackupRepository['id'][] }
}
vms?: {
id: XoVm['id'] | { __or: XoVm['id'][] } | Record<string, unknown>
}
srs: {
id: XoSr['id'] | { __or: XoSr['id'][] }
}
type: BACKUP_TYPE
settings: {
'': {
cbtDestroySnapshotData?: boolean
concurrency?: number
longTermRetention?: {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
maxExportRate?: number
nbdConcurrency?: number
nRetriesVmBackupFailures?: number
preferNbd?: boolean
timezone?: string
[key: string]: unknown
}
[key: XoSchedule['id']]: {
exportRetention?: number
healthCheckSr?: XoSr['id']
healthCheckVmsWithTags?: string[]
fullInterval?: number
copyRetention?: number
snapshotRetention?: number
cbtDestroySnapshotData?: boolean
[key: string]: unknown
}
}
}
export type XoJob = BaseXoJob & {}
export type XoSchedule = {
cron: string
enabled: boolean
enable: boolean
id: Branded<'schedule'>
jobId: (XoJob | XoBackupJob)['id']
jobId: XoJob['id']
name?: string
timezone?: string
}
@@ -440,7 +322,6 @@ export type XoServer = {
httpProxy?: string
id: Branded<'server'>
label?: string
master?: XoHost['id']
poolId?: XoPool['id']
poolNameDescription?: string
poolNameLabel?: string
@@ -454,7 +335,7 @@ export type XoSr = BaseXapiXo & {
$container: XoPool['id'] | XoHost['id']
VDIs: AnyXoVdi['id'][]
VDIs: XoVdi['id'][]
allocationStrategy: 'thin' | 'thick' | 'unknown'
content_type: string
@@ -464,9 +345,9 @@ export type XoSr = BaseXapiXo & {
name_description: string
name_label: string
other_config: Record<string, string>
physical_usage: number
physical_usage: number | null
shared: boolean
size: number
size: number | null
sm_config: Record<string, string>
SR_type: string
tags: string[]
@@ -474,21 +355,6 @@ export type XoSr = BaseXapiXo & {
usage: number
}
export type XoSm = BaseXapiXo & {
id: Branded<'SM'>
SM_type: string
vendor: string
name_label: string
name_description: string
configuration: Record<string, string>
features: Record<string, number>
driver_filename: string
required_cluster_stack: string[]
supported_image_formats: string[]
type: 'SM'
}
export type XoUser = {
authProviders?: Record<string, string>
email: string
@@ -509,8 +375,8 @@ export type XoVbd = BaseXapiXo & {
position: string
read_only: boolean
type: 'VBD'
VDI?: XoVdi['id'] | XoVdiSnapshot['id']
VM: AnyXoVm['id']
VDI: XoVdi['id']
VM: XoVm['id']
}
type BaseXoVdi = BaseXapiXo & {
@@ -526,7 +392,6 @@ type BaseXoVdi = BaseXapiXo & {
name_label: string
other_config: Record<string, string>
parent?: XoVdiUnmanaged['id']
image_format?: string
size: number
snapshots: XoVdiSnapshot['id'][]
tags: string[]
@@ -545,24 +410,14 @@ export type XoVdiSnapshot = BaseXoVdi & {
type: 'VDI-snapshot'
}
export type XoVdiUnmanaged = Omit<BaseXoVdi, '$VBDs'> & {
export type XoVdiUnmanaged = BaseXoVdi & {
id: Branded<'VDI-unmanaged'>
type: 'VDI-unmanaged'
/**
* Unmanaged VDI have no VBDs
*/
$VBDs: never[]
}
export type XoVgpu = BaseXapiXo & {
id: Branded<'VGPU'>
type: 'vgpu'
}
export type XoVgpuType = BaseXapiXo & {
id: Branded<'vgpu-type'>
type: 'vgpuType'
type: 'VGPU'
}
export type XoVif = BaseXapiXo & {
@@ -590,7 +445,6 @@ export type XoVif = BaseXapiXo & {
export type XoVm = BaseXoVm & {
id: Branded<'VM'>
type: 'VM'
vulnerabilities: { xsa468: boolean | { reason: string; driver?: string; version?: string } }
}
export type XoVmController = BaseXoVm & {
@@ -631,13 +485,9 @@ export type XoVtpm = BaseXapiXo & {
export type XapiXoRecord =
| XoAlarm
| XoGpuGroup
| XoHost
| XoMessage
| XoNetwork
| XoPci
| XoPgpu
| XoPif
| XoPool
| XoSr
| XoVbd
@@ -645,21 +495,13 @@ export type XapiXoRecord =
| XoVdiSnapshot
| XoVdiUnmanaged
| XoVgpu
| XoVgpuType
| XoVif
| XoVm
| XoVmController
| XoVmSnapshot
| XoVmTemplate
| XoVtpm
| XoSm
export type NonXapiXoRecord = XoGroup | XoProxy | XoJob | XoBackupRepository | XoSchedule | XoServer | XoUser
export type NonXapiXoRecord = XoGroup | XoJob | XoSchedule | XoServer | XoUser
export type XoRecord = XapiXoRecord | NonXapiXoRecord
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
export type AnyXoVdi = XoVdi | XoVdiSnapshot | XoVdiUnmanaged
export type AnyXoJob = XoJob | XoBackupJob

View File

@@ -20,7 +20,7 @@ function main([inputPath = 0]) {
try {
data = JSON.parse(input)
} catch (error) {
// fall back to XML
// fallback to XML
log(parseXml(input))
return
}

View File

@@ -4,10 +4,6 @@ Usage:
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
Detects and repair issues with VM backups.
Options:

View File

@@ -20,10 +20,6 @@ Usage:
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
Detects and repair issues with VM backups.
Options:

View File

@@ -1,81 +1,13 @@
import { asyncEach } from '@vates/async-each'
import { asyncMap } from '@xen-orchestra/async-map'
import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
import { getSyncedHandler } from '@xen-orchestra/fs'
import getopts from 'getopts'
import { basename, dirname, join } from 'path'
import { basename, dirname } from 'path'
import Disposable from 'promise-toolbox/Disposable'
import { pathToFileURL } from 'url'
async function cleanFiles(vmDirs, { fix, merge, remove }) {
let nbSucceed = 0
let nbFailed = 0
let nbStarted = 0
const total = vmDirs.length
// don't assume all the files uses the same remote
await asyncEach(
vmDirs,
vmDir =>
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
try {
nbStarted++
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
nbSucceed++
} catch (error) {
nbFailed++
console.error('adapter.cleanVm', vmDir, error, { nbStarted, nbSucceed, nbFailed, total })
}
}),
{ concurrency: 2 }
)
console.log('done', { nbSucceed, nbFailed })
}
async function cleanRemote(url, { fix, merge, remove }) {
let nbSucceed = 0
let nbFailed = 0
let nbStarted = 0
await Disposable.use(getSyncedHandler({ url }), async handler => {
const remoteAdapter = new RemoteAdapter(handler)
const vmUuids = await remoteAdapter.listAllVms()
const total = vmUuids.length
await asyncEach(
vmUuids,
async vmUuid => {
try {
nbStarted++
await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
nbSucceed++
} catch (error) {
nbFailed++
console.error('adapter.cleanVm', vmUuid, error, { nbStarted, nbSucceed, nbFailed, total })
}
},
{ concurrency: 2 }
)
})
console.log('done', { nbSucceed, nbFailed })
}
export default async function cleanVms(args) {
const {
_: vmDirs,
fix,
remote,
remove,
merge,
} = getopts(args, {
const { _, fix, remove, merge } = getopts(args, {
alias: {
fix: 'f',
remove: 'r',
@@ -88,9 +20,19 @@ export default async function cleanVms(args) {
},
})
if (remote) {
return cleanRemote(remote, { fix, merge, remove })
} else {
return cleanFiles(vmDirs, { fix, merge, remove })
}
await asyncMap(_, vmDir =>
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
try {
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
logWarn: (...args) => console.warn(...args),
})
} catch (error) {
console.error('adapter.cleanVm', vmDir, error)
}
})
)
}

View File

@@ -6,10 +6,9 @@
"preferGlobal": true,
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@vates/async-each": "^1.0.0",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.63.0",
"@xen-orchestra/fs": "^4.6.2",
"@xen-orchestra/backups": "^0.59.0",
"@xen-orchestra/fs": "^4.5.0",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",
"lodash": "^4.17.15",
@@ -28,7 +27,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "1.1.2",
"version": "1.0.30",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -4,13 +4,12 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { importIncrementalVm } from './_incrementalVm.mjs'
import { Task } from './Task.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { VhdNegative, VhdSynthetic } from 'vhd-lib'
import { decorateClass } from '@vates/decorate-with'
import { createLogger } from '@xen-orchestra/log'
import { dirname, join } from 'node:path'
import pickBy from 'lodash/pickBy.js'
import { defer } from 'golike-defer'
import { NegativeDisk } from '@xen-orchestra/disk-transform'
import { openDiskChain } from './disks/openDiskChain.mjs'
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
async function resolveUuid(xapi, cache, uuid, type) {
@@ -60,7 +59,7 @@ export class ImportVmBackup {
const metadata = this._metadata
const { mapVdisSrs } = this._importIncrementalVmSettings
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
const disks = {}
const streams = {}
const metdataDir = dirname(metadata._filename)
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
@@ -111,21 +110,20 @@ export class ImportVmBackup {
}
}
let disk
let stream
const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
if (vhdPath === backupWithSnapshotPath) {
// all the data are already on the host
debug('direct reuse of a snapshot')
disk = null
stream = null
vdis[vdiRef].baseVdi = snapshotCandidate
// go next disk , we won't use this stream
continue
}
const parent = await openDiskChain({
handler: this._adapter._handler,
path: vhdPath,
})
let disposableDescendants
const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
// this will also clean if another disk of this VM backup fails
// if user really only need to restore non failing disks he can retry with ignoredVdis
@@ -134,7 +132,8 @@ export class ImportVmBackup {
if (!disposed) {
disposed = true
try {
await parent?.close()
await disposableDescendants?.dispose()
await disposableSynthetic?.dispose()
} catch (error) {
warn('openVhd: failed to dispose VHDs', { error })
}
@@ -142,10 +141,11 @@ export class ImportVmBackup {
}
$defer.onFailure(() => disposeOnce())
debug('got vhd synthetic of parents', parent)
const parentVhd = disposableSynthetic.value
await parentVhd.readBlockAllocationTable()
debug('got vhd synthetic of parents', parentVhd.length)
if (snapshotCandidate !== undefined) {
let descendant, negativeDisk
try {
debug('will try to use differential restore', {
backupWithSnapshotPath,
@@ -153,37 +153,39 @@ export class ImportVmBackup {
vdiRef,
})
descendant = await openDiskChain({
handler: this._adapter._handler,
path: backupWithSnapshotPath,
disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
until: vhdPath,
})
const descendantsVhd = disposableDescendants.value
await descendantsVhd.readBlockAllocationTable()
debug('got vhd synthetic of descendants')
negativeDisk = new NegativeDisk(parent, descendant)
const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
debug('got vhd negative')
// update the stream with the negative vhd stream
disk = negativeDisk
stream = await negativeVhd.stream()
vdis[vdiRef].baseVdi = snapshotCandidate
} catch (error) {
// can be a broken VHD chain, a vhd chain with a key backup, ....
// not an irrecuperable error, don't dispose parentVhd, and fall back to full restore
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
warn(`can't use differential restore`, { error })
descendant?.close()
negativeDisk?.close()
disposableDescendants?.dispose()
}
}
// didn't make a negative stream : fall back to classic stream
if (disk === undefined) {
// didn't make a negative stream : fallback to classic stream
if (stream === undefined) {
debug('use legacy restore')
disk = parent
stream = await parentVhd.stream()
}
info('everything is ready, will transfer', disk)
disks[vdiRef] = disk
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
info('everything is ready, will transfer', stream.length)
streams[`${vdiRef}.vhd`] = stream
}
return {
disks,
streams,
vbds,
vdis,
version: '1.0.0',
@@ -239,6 +241,7 @@ export class ImportVmBackup {
assert.strictEqual(metadata.mode, 'delta')
backup = await this.#decorateIncrementalVmMetadata()
Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
}
return Task.run(

View File

@@ -2,7 +2,7 @@ import { asyncEach } from '@vates/async-each'
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import { compose } from '@vates/compose'
import { createLogger } from '@xen-orchestra/log'
import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
import { decorateMethodsWith } from '@vates/decorate-with'
import { deduped } from '@vates/disposable/deduped.js'
import { dirname, join, resolve } from 'node:path'
@@ -10,6 +10,7 @@ import { execFile } from 'child_process'
import { mount } from '@vates/fuse-vhd'
import { readdir, lstat } from 'node:fs/promises'
import { synchronized } from 'decorator-synchronized'
import { v4 as uuidv4 } from 'uuid'
import { ZipFile } from 'yazl'
import Disposable from 'promise-toolbox/Disposable'
import fromCallback from 'promise-toolbox/fromCallback'
@@ -31,11 +32,6 @@ import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
import { lvs, pvs } from './_lvm.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { RemoteVhd } from './disks/RemoteVhd.mjs'
import { openDiskChain } from './disks/openDiskChain.mjs'
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
import { ReadAhead } from '@xen-orchestra/disk-transform'
export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
@@ -205,7 +201,7 @@ export class RemoteAdapter {
})
}
// check if we will be allowed to merge a vhd created in this adapter
// check if we will be allowed to merge a a vhd created in this adapter
// with the vhd at path `path`
async isMergeableParent(packedParentUid, path) {
return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
@@ -693,24 +689,22 @@ export class RemoteAdapter {
return path
}
async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
const handler = this._handler
if (this.useVhdDirectory()) {
await writeToVhdDirectory({
disk,
target: {
handler,
path,
concurrency: writeBlockConcurrency,
validator,
compression: 'brotli',
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
concurrency: writeBlockConcurrency,
compression: this.#getCompressionType(),
async validator() {
await input.task
return validator.apply(this, arguments)
},
})
await VhdAbstract.createAlias(handler, path, dataPath)
return size
} else {
const stream = await toVhdStream(disk)
await this.outputStream(path, stream, { validator })
await validator(path)
return this.outputStream(path, input, { checksum, validator })
}
}
@@ -734,16 +728,30 @@ export class RemoteAdapter {
}
// open the hierarchy of ancestors until we find a full one
async _createVhdDisk(handler, path, { useChain }) {
let disk
if (useChain) {
disk = await openDiskChain({ handler, path })
} else {
disk = new RemoteVhd({ handler, path })
await disk.init()
async _createVhdStream(handler, path, { useChain }) {
const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
// I don't want the vhds to be disposed on return
// but only when the stream is done ( or failed )
let disposed = false
const disposeOnce = async () => {
if (!disposed) {
disposed = true
try {
await disposableSynthetic.dispose()
} catch (error) {
warn('openVhd: failed to dispose VHDs', { error })
}
}
}
disk = new ReadAhead(disk)
return disk
const synthetic = disposableSynthetic.value
await synthetic.readBlockAllocationTable()
const stream = await synthetic.stream()
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
return stream
}
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
@@ -751,14 +759,14 @@ export class RemoteAdapter {
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
const dir = dirname(metadata._filename)
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
const disks = {}
const streams = {}
await asyncMapSettled(Object.keys(vdis), async ref => {
delete vdis[ref].baseVdi
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
})
return {
disks,
streams,
vbds,
vdis,
version: '1.0.0',
@@ -776,7 +784,7 @@ export class RemoteAdapter {
let json
let isImmutable = false
let remoteIsImmutable = false
// if the remote is immutable, check if this metadata is also immutable
// if the remote is immutable, check if this metadatas are also immutables
try {
// this file is not encrypted
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
@@ -792,7 +800,7 @@ export class RemoteAdapter {
json = await this.handler.readFile(path, { flag: 'r+' })
// s3 handler don't respect flags
} catch (err) {
// retry without triggering immutability check ,only on immutable remote
// retry without triggerring immutbaility check ,only on immutable remote
if (err.code === 'EPERM' && remoteIsImmutable) {
isImmutable = true
json = await this._handler.readFile(path, { flag: 'r' })

View File

@@ -24,7 +24,7 @@ export class RestoreMetadataBackup {
const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
const data = await handler.readFile(dataFileName)
// if data is JSON, sent it as a plain string; otherwise, consider the data as binary and encode it
// if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
const isJson = dataFileName.endsWith('.json')
return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
}

View File

@@ -132,7 +132,7 @@ test('it remove vhd with missing or multiple ancestors', async () => {
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
// we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
})
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
@@ -287,7 +287,7 @@ test('it finish unterminated merge ', async () => {
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
// the message an resulting files should be identical to the output with vhd files which is tested independantly
describe('tests multiple combination ', { concurrency: 1 }, () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
@@ -308,9 +308,9 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
// a vhd non referenced in metada
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
// an abandonded delta vhd without its parent
await generateVhd(`${basePath}/abandonned.vhd`, {
useAlias,
mode: vhdMode,
@@ -404,7 +404,7 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
@@ -454,7 +454,7 @@ test('check Aliases should work alone', async () => {
logWarn: () => {},
})
// only ok have survived
// only ok have suvived
const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
assert.equal(alias.length, 1)

View File

@@ -512,7 +512,7 @@ export async function cleanVm(
remove,
mergeBlockConcurrency,
})
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
})
}
@@ -595,7 +595,7 @@ export async function cleanVm(
// all disks are now key disk
metadata.isVhdDifferencing = {}
for (const id of Object.keys(metadata.vdis ?? {})) {
metadata.isVhdDifferencing[id] = false
metadata.isVhdDifferencing[`${id}.vhd`] = false
}
}
mustRegenerateCache = true

View File

@@ -26,7 +26,7 @@ const LTR_DEFINITIONS = {
copy.date(date.date() - firstDayOfWeek)
// warning, the year in term of week may different from YYYY
// since the computation of the first week of a year is timezone dependent
// since the computation of the first week of a year is timezone dependant
return copy.format('gggg-ww')
}
},
@@ -59,7 +59,7 @@ const LTR_DEFINITIONS = {
* return the entries too old to be kept
* if multiple entries are i the same time bucket : keep only the most recent one
* if an entry is valid in any of the bucket OR the minRetentionCount : keep it
* if a bucket is completely empty : it does not count as one, thus it may extend the retention
* if a bucket is completly empty : it does not count as one, thus it may extend the retention
* @returns Array<Backup>
*/
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {

View File

@@ -10,11 +10,6 @@ import { Task } from './Task.mjs'
import pick from 'lodash/pick.js'
import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
import { XapiDiskSource } from '@xen-orchestra/xapi'
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
import { toQcow2Stream } from '@xen-orchestra/qcow2'
import { VHD_MAX_SIZE } from '@xen-orchestra/xapi/disks/Xapi.mjs'
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
export async function exportIncrementalVm(
@@ -24,7 +19,7 @@ export async function exportIncrementalVm(
) {
// refs of VM's VDIs → base's VDIs.
const disks = {}
const streams = {}
const vdis = {}
const vbds = {}
await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
@@ -57,14 +52,32 @@ export async function exportIncrementalVm(
$snapshot_of$uuid: vdi.$snapshot_of?.uuid,
$SR$uuid: vdi.$SR.uuid,
}
disks[vdiRef] = new XapiDiskSource({
vdiRef,
xapi: vm.$xapi,
baseRef: baseVdi?.$ref,
nbdConcurrency,
preferNbd,
})
await disks[vdiRef].init()
try {
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
baseRef: baseVdi?.$ref,
cancelToken,
format: 'vhd',
nbdConcurrency,
preferNbd,
})
} catch (err) {
if (err.code === 'VDI_CANT_DO_DELTA') {
// fall back to a base
Task.info(`Can't do delta, will try to get a full stream`, { vdi })
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
cancelToken,
format: 'vhd',
nbdConcurrency,
preferNbd,
})
// only warn if the fall back succeed
Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
vdi,
})
} else {
throw err
}
}
})
const suspendVdi = vm.$suspend_VDI
@@ -74,13 +87,10 @@ export async function exportIncrementalVm(
...suspendVdi,
$SR$uuid: suspendVdi.$SR.uuid,
}
disks[vdiRef] = new XapiDiskSource({
vdiRef: suspendVdi.$ref,
xapi: vm.$xapi,
nbdConcurrency,
preferNbd,
streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
cancelToken,
format: 'vhd',
})
await disks[vdiRef].init()
}
const vifs = {}
@@ -106,17 +116,24 @@ export async function exportIncrementalVm(
})
)
return {
version: '1.1.0',
vbds,
vdis,
vifs,
vm: {
...vm,
return Object.defineProperty(
{
version: '1.1.0',
vbds,
vdis,
vifs,
vm: {
...vm,
},
vtpms,
},
vtpms,
disks,
}
'streams',
{
configurable: true,
value: streams,
writable: true,
}
)
}
export const importIncrementalVm = defer(async function importIncrementalVm(
@@ -182,8 +199,8 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
const vdi = vdiRecords[vdiRef]
let newVdi
if (vdi.baseVdi?.$ref !== undefined) {
newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
if (vdi.baseVdi !== undefined) {
newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
$defer.onFailure(() => newVdi.$destroy())
await newVdi.update_other_config(COPY_OF, vdi.uuid)
@@ -227,28 +244,21 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
}
})
const { disks } = incrementalVm
const { streams } = incrementalVm
await Promise.all([
// Import VDI contents.
cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
for (const disk of ensureArray(disks[id])) {
if (disk === null) {
// we restore a backup and reuse completely a local snapshot
for (let stream of ensureArray(streams[`${id}.vhd`])) {
if (stream === null) {
// we restore a backup and reuse completly a local snapshot
continue
}
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
let stream, format
if (vdi.virtual_size > VHD_MAX_SIZE) {
stream = await toQcow2Stream(disk)
format = 'qcow2'
} else {
stream = await toVhdStream(disk)
format = 'vhd'
if (typeof stream === 'function') {
stream = await stream()
}
await vdi.$importContent(stream, { cancelToken, format })
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
}
}),

View File

@@ -44,7 +44,7 @@ const isValidTar = async (handler, size, fd) => {
return buf.every(_ => _ === 0)
}
// TODO: find a heuristic for compressed files
// TODO: find an heuristic for compressed files
export async function isValidXva(path) {
const handler = this._handler

View File

@@ -43,7 +43,7 @@ async function getDeltaChainLength(xapi, type, ref) {
}
/**
* set the delta chain length ( number of delta since last base backup) to a VM and its associated VDIs
* set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
@@ -58,7 +58,7 @@ export async function setVmDeltaChainLength(xapi, vmRef, length) {
/**
* Compute the delta chain length of a VM and its associated VDIs
* if there is a discrepancy, use, the highest value
* if there is a discrependcy, use, the highest value
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
@@ -92,7 +92,7 @@ export function resetVmOtherConfig(xapi, vmRef) {
/**
*
* used to ensure compatibility with the previous snapshots that were having the config stored only into VM
* used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
*
* @param {Xapi} xapi
* @param {String} vmRef
@@ -152,7 +152,7 @@ export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, schedule
}
/**
*
* mark the export of he VM and its VDIs as successful
* mark the export of he VM and its VDIs as successfull
*
* @param {Xapi} xapi
* @param {String} vmRef

View File

@@ -4,11 +4,11 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
import { Throttle } from '@vates/generator-toolbox'
const noop = Function.prototype
@@ -41,7 +41,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const schedule = this._schedule
const settings = this._settings
const throttleGenerator = new Throttle()
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
@@ -89,7 +89,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
schedule,
settings: vmSettings,
sourceRemoteAdapter,
throttleGenerator,
throttleStream,
vmUuid,
}
let vmBackup

View File

@@ -4,12 +4,12 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { runTask } from './_runTask.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
import { FullXapi } from './_vmRunners/FullXapi.mjs'
import { Throttle } from '@vates/generator-toolbox'
const noop = Function.prototype
@@ -55,7 +55,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const schedule = this._schedule
const settings = this._settings
const throttleGenerator = new Throttle()
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
@@ -147,7 +147,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
schedule,
settings: vmSettings,
srs,
throttleGenerator,
throttleStream,
vm,
}

View File

@@ -24,9 +24,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
writer =>
writer.run({
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additional properties to it
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
streamLength: stream.length,
maxStreamLength: stream.maxStreamLength, // on encrypted source
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,

View File

@@ -13,35 +13,15 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
return [FullRemoteWriter, FullXapiWriter]
}
async _mustDoSnapshot() {
_mustDoSnapshot() {
const vm = this._vm
const settings = this._settings
// General setting forcing snapshot at all times
if (settings.unconditionalSnapshot) {
return true
}
// If offline backup option is not checked
// and vm is running, we make a snapshot
if (!settings.offlineBackup && vm.power_state === 'Running') {
return true
}
// General setting used for snapshot retention
// see schedule in backupJob edition
if (settings.snapshotRetention !== 0) {
return true
}
const vdis = await this._xapi.getRecords('VDI', await vm.$getDisks())
const hasNOBAKtag = vdis.some(idx => idx.name_label.includes('[NOBAK]'))
// if there is a disk with NOBAK tag
// we need to make a snapshot
if (hasNOBAKtag) {
return true
}
return (
settings.unconditionalSnapshot ||
(!settings.offlineBackup && vm.power_state === 'Running') ||
settings.snapshotRetention !== 0
)
}
_selectBaseVm() {}
@@ -49,24 +29,22 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
const { compression } = this.job
const vm = this._vm
const exportedVm = this._exportedVm
// @todo put back throttle for full backup/Replication
const stream =
/* this._throttleStream( */
const stream = this._throttleStream(
(
await this._xapi.VM_export(exportedVm.$ref, {
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
useSnapshot: false,
})
).body
/* ) */
)
const vdis = await exportedVm.$getDisks()
let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe
for (const vdiRef of vdis) {
const vdi = await this._xapi.getRecord('VDI', vdiRef)
// the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
// of the real stream size in general, since a disk is never completely full
// the size a of fully allocated vdi will be virtual_size exaclty, it's a gross over evaluation
// of the real stream size in general, since a disk is never completly full
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
maxStreamLength += vdi.virtual_size
}

View File

@@ -3,6 +3,8 @@ import { createLogger } from '@xen-orchestra/log'
import { asyncEach } from '@vates/async-each'
import assert from 'node:assert'
import * as UUID from 'uuid'
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
import mapValues from 'lodash/mapValues.js'
import { AbstractRemote } from './_AbstractRemote.mjs'
import { forkDeltaExport } from './_forkDeltaExport.mjs'
@@ -10,7 +12,6 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
import { Disposable } from 'promise-toolbox'
import { openVhd } from 'vhd-lib'
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
const { warn } = createLogger('xo:backups:Incrementalremote')
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
@@ -18,7 +19,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
return IncrementalRemoteWriter
}
// we'll transfer the full list if at least one backup should be transferred
// we'll transfer the full list if at least one backup should be transfered
// to ensure we don't cut the delta chain
_filterTransferList(transferList) {
if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
@@ -36,7 +37,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
return
}
await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
const isDifferencing = metadata.isVhdDifferencing[id]
const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
if (isDifferencing) {
const vmDir = getVmBackupDir(metadata.vm.uuid)
const path = `${vmDir}/${metadata.vhds[id]}`
@@ -70,12 +71,9 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
// recompute if disks are differencing or not
const isVhdDifferencing = {}
for (const key in incrementalExport.disks) {
const disk = incrementalExport.disks[key]
isVhdDifferencing[key] = disk.isDifferencing()
incrementalExport.disks[key] = new SynchronizedDisk(disk)
}
await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
})
const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
if (metadata.isBase === hasDifferencingDisk) {
warn(`Metadata isBase and real disk value are different`, {
@@ -89,10 +87,11 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
await this._selectBaseVm(metadata)
await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
await this._callWriters(
writer =>
writer.transfer({
deltaExport: forkDeltaExport(incrementalExport, writer.constructor.name),
deltaExport: forkDeltaExport(incrementalExport),
isVhdDifferencing,
timestamp: metadata.timestamp,
vm: metadata.vm,

View File

@@ -1,12 +1,18 @@
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { pipeline } from 'node:stream'
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
import keyBy from 'lodash/keyBy.js'
import mapValues from 'lodash/mapValues.js'
import vhdStreamValidator from 'vhd-lib/vhdStreamValidator.js'
import { AbstractXapi } from './_AbstractXapi.mjs'
import { forkDeltaExport } from './_forkDeltaExport.mjs'
import { exportIncrementalVm } from '../../_incrementalVm.mjs'
import { forkDeltaExport } from './_forkDeltaExport.mjs'
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
import { Task } from '../../Task.mjs'
import { watchStreamSize } from '../../_watchStreamSize.mjs'
import {
DATETIME,
DELTA_CHAIN_LENGTH,
@@ -14,16 +20,16 @@ import {
setVmDeltaChainLength,
markExportSuccessfull,
} from '../../_otherConfig.mjs'
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
const noop = Function.prototype
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
_getWriters() {
return [IncrementalRemoteWriter, IncrementalXapiWriter]
}
async _mustDoSnapshot() {
_mustDoSnapshot() {
return true
}
@@ -39,33 +45,42 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
nbdConcurrency: this._settings.nbdConcurrency,
preferNbd: this._settings.preferNbd,
})
const isVhdDifferencing = {}
let useNbd = false
for (const key in deltaExport.disks) {
const disk = deltaExport.disks[key]
isVhdDifferencing[key] = disk.isDifferencing()
deltaExport.disks[key] = new SynchronizedDisk(disk)
useNbd = useNbd || disk.useNbd()
}
if (useNbd) {
// since NBD is network based, if one disk use nbd , all the disk use them
// except the suspended VDI
if (Object.values(deltaExport.streams).some(({ _nbd }) => _nbd)) {
Task.info('Transfer data using NBD')
}
// @todo : reimplement throttle
const isVhdDifferencing = {}
// since isVhdDifferencingDisk is reading and unshifting data in stream
// it should be done BEFORE any other stream transform
await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
})
const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
if (this._settings.validateVhdStreams) {
deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
}
deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
const timestamp = Date.now()
await this._callWriters(
writer =>
writer.transfer({
deltaExport: forkDeltaExport(deltaExport, writer.constructor.name),
deltaExport: forkDeltaExport(deltaExport),
isVhdDifferencing,
sizeContainers,
timestamp,
vm,
vmSnapshot: exportedVm,
}),
'writer.transfer()'
)
// we want to control the uuid of the vhd in the chain
// and ensure they are correctly chained
await this._callWriters(
writer =>
writer.updateUuidAndChain({
@@ -87,6 +102,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
await markExportSuccessfull(this._xapi, exportedVm.$ref)
}
const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
const end = Date.now()
const duration = end - timestamp
debug('transfer complete', {
duration,
speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
size,
})
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
}

View File

@@ -21,7 +21,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
schedule,
settings,
sourceRemoteAdapter,
throttleGenerator,
throttleStream,
vmUuid,
}) {
super()
@@ -34,7 +34,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this._healthCheckSr = healthCheckSr
this._sourceRemoteAdapter = sourceRemoteAdapter
this._throttleGenerator = throttleGenerator
this._throttleStream = throttleStream
this._vmUuid = vmUuid
const allSettings = job.settings
@@ -90,7 +90,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
for (const timestamp of timestamps) {
if (remoteMetadatas[timestamp] !== nbRemotes) {
// this backup is not present in all the remote
// should be retransferred if not found later
// should be retransfered if not found later
transferList.push(localMetada.get(timestamp))
} else {
// backup is present in local and remote : the chain has already been transferred

View File

@@ -28,7 +28,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
schedule,
settings,
srs,
throttleGenerator,
throttleStream,
vm,
}) {
super()
@@ -62,7 +62,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
this._healthCheckSr = healthCheckSr
this._jobId = job.id
this._jobSnapshotVdis = undefined
this._throttleGenerator = throttleGenerator
this._throttleStream = throttleStream
this._xapi = vm.$xapi
// Base VM for the export
@@ -141,14 +141,14 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
const settings = this._settings
if (await this._mustDoSnapshot()) {
if (this._mustDoSnapshot()) {
await Task.run({ name: 'snapshot' }, async () => {
if (!settings.bypassVdiChainsCheck) {
await vm.$assertHealthyVdiChains()
}
if (settings.cbtDestroySnapshotData) {
if (settings.preferNbd) {
try {
// enable CBT on all disks if we want to be able to purge snapshot data
// enable CBT on all disks if possible
const diskRefs = await xapi.VM_getDisks(vm.$ref)
await Promise.all(diskRefs.map(diskRef => xapi.callAsync('VDI.enable_cbt', diskRef)))
} catch (error) {
@@ -195,7 +195,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
}
})
await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
// end of compatibility handling
// end of compatibiliy handling
// handle snapshot by VDI
this._jobSnapshotVdis = []
@@ -298,7 +298,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
// preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
// in that case next runs will be full, but there is not an easy way to prevent that
this._settings.preferNbd &&
// only delete snapshot data if the config allows it
// only delete snapshost data if the config allows it
this._settings.cbtDestroySnapshotData
) {
Task.info('will delete snapshot data')
@@ -324,7 +324,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
throw new Error('Not implemented')
}
async _mustDoSnapshot() {
_mustDoSnapshot() {
throw new Error('Not implemented')
}

View File

@@ -1,16 +1,11 @@
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
import cloneDeep from 'lodash/cloneDeep.js'
import mapValues from 'lodash/mapValues.js'
export function forkDeltaExport(deltaExport, label) {
label += ' ' + Math.random()
const { disks, ...rest } = deltaExport
const fork = cloneDeep(rest)
fork.disks = {}
for (const [key, disk] of Object.entries(disks)) {
if (!(disk instanceof SynchronizedDisk)) {
throw new Error('Can only fork synchronized disks ')
}
fork.disks[key] = disk.fork(label)
}
return fork
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
export function forkDeltaExport(deltaExport) {
const { streams, ...rest } = deltaExport
const newMetadata = cloneDeep(rest)
newMetadata.streams = mapValues(streams, forkStreamUnpipe)
return newMetadata
}

View File

@@ -136,7 +136,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
const { handler } = this._adapter
const vhds = this.#vhds
await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
const isDifferencing = isVhdDifferencing[id]
const isDifferencing = isVhdDifferencing[`${id}.vhd`]
const path = `${this._vmBackupDir}/${vhds[id]}`
if (isDifferencing) {
assert.notStrictEqual(
@@ -202,8 +202,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// skip backup while being vigilant to not stuck the forked disk
await Promise.all(Object.values(deltaExport.disks).map(async disk => disk.close()))
// skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
Object.values(deltaExport.streams).forEach(stream => stream.destroy())
return { size: 0 }
}
@@ -222,33 +223,36 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
vmSnapshot,
vtpms: deltaExport.vtpms,
}
let size = 0
await Task.run({ name: 'transfer' }, async () => {
const { size } = await Task.run({ name: 'transfer' }, async () => {
let transferSize = 0
await asyncEach(
Object.entries(deltaExport.disks),
async ([diskRef, disk]) => {
const path = `${this._vmBackupDir}/${vhds[diskRef]}`
await adapter.writeVhd(path, disk, {
Object.keys(deltaExport.vdis),
async id => {
const path = `${this._vmBackupDir}/${vhds[id]}`
// don't write it as transferSize += await async function
// since i += await asyncFun lead to race condition
// as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
// no checksum for VHDs, because they will be invalidated by
// merges and chains
// merges and chainings
checksum: false,
validator: tmpPath => checkVhd(handler, tmpPath),
writeBlockConcurrency: this._config.writeBlockConcurrency,
})
size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
transferSize += transferSizeOneDisk
},
{
concurrency: settings.diskPerVmConcurrency,
}
)
return { size }
return { size: transferSize }
})
metadataContent.size = size // @todo return exactly the size written by this writer
metadataContent.size = size
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
// TODO: run cleanup?
return { size }
}
}
decorateClass(IncrementalRemoteWriter, {

View File

@@ -21,7 +21,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
}
// @todo use an index if possible
// @todo : this seems similar to decorateVmMetadata
// @todo : this seems similare to decorateVmMetadata
const replicatedVdis = sr.$VDIs
.filter(vdi => {
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
@@ -130,7 +130,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
return backup
}
async _transfer({ timestamp, deltaExport, vm }) {
async _transfer({ timestamp, deltaExport, sizeContainers, vm }) {
const { _warmMigration } = this._settings
const sr = this._sr
const job = this._job
@@ -141,12 +141,8 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
let targetVmRef
await Task.run({ name: 'transfer' }, async () => {
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
// size is mandatory to ensure the task have the right data
return {
size: Object.values(deltaExport.disks).reduce(
(sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
0
),
size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
}
})
this._targetVmRef = targetVmRef

View File

@@ -20,14 +20,10 @@ export class AbstractIncrementalWriter extends AbstractWriter {
async transfer({ deltaExport, ...other }) {
try {
return await this._transfer({ deltaExport, ...other })
} catch (err) {
console.error({ err })
} finally {
// ensure all sources are properly closed
for (const disk of Object.values(deltaExport.disks)) {
try {
await disk.close()
} catch (err) {}
// ensure all streams are properly closed
for (const stream of Object.values(deltaExport.streams)) {
stream.destroy()
}
}
}

View File

@@ -1,171 +0,0 @@
// @ts-check
/**
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
* @typedef {import('@xen-orchestra/disk-transform').Disk} Disk
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
*/
import { openVhd } from 'vhd-lib'
import { DISK_TYPES } from 'vhd-lib/_constants.js'
import { dirname, join } from 'node:path'
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
/**
* Represents a remote VHD (Virtual Hard Disk) that extends RandomAccessDisk.
*/
export class RemoteVhd extends RandomAccessDisk {
/**
* @type {string}
*/
#path
/**
* @type {FileAccessor}
*/
#handler
/**
* @type {VhdFile | VhdDirectory | undefined}
*/
#vhd
/**
* @type {boolean | undefined}
*/
#isDifferencing
/**
* @type {() => any}
*/
#dispose = () => {}
/**
* @returns {string}
*/
get path() {
return this.#path
}
/**
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {string} params.path
*/
constructor({ handler, path }) {
super()
// @todo : ensure this is the full path from the root of the remote
this.#path = path
this.#handler = handler
}
/**
* @returns {number}
*/
getVirtualSize() {
if (this.#vhd === undefined) {
throw new Error(`can't call getvirtualsize of a RemoteVhd before init`)
}
return this.#vhd.footer.currentSize
}
/**
* @returns {number}
*/
getBlockSize() {
return 2 * 1024 * 1024
}
/**
* Initializes the VHD.
* @returns {Promise<void>}
*/
async init() {
const { value, dispose } = await openVhd(this.#handler, this.#path)
this.#vhd = value
this.#dispose = dispose
await this.#vhd.readBlockAllocationTable()
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
}
/**
* Closes the VHD.
* @returns {Promise<void>}
*/
async close() {
await this.#dispose()
}
/**
* Checks if the VHD contains a specific block.
* @param {number} index
* @returns {boolean}
*/
hasBlock(index) {
if (this.#vhd === undefined) {
throw new Error(`can't call hasblock of a RemoteVhd before init`)
}
return this.#vhd.containsBlock(index)
}
/**
* Gets the indexes of all blocks in the VHD.
* @returns {Array<number>}
*/
getBlockIndexes() {
if (this.#vhd === undefined) {
throw new Error(`can't call getBlockIndexes of a RemoteVhd before init`)
}
const index = []
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
if (this.hasBlock(blockIndex)) {
index.push(blockIndex)
}
}
return index
}
/**
* Reads a specific block from the VHD.
* @param {number} index
* @returns {Promise<DiskBlock>}
*/
async readBlock(index) {
if (this.#vhd === undefined) {
throw new Error(`can't call readBlock of a RemoteVhd before init`)
}
const { data } = await this.#vhd.readBlock(index)
return {
index,
data,
}
}
/**
*
* @returns {RandomAccessDisk}
*/
instantiateParent() {
if (this.#vhd === undefined) {
throw new Error(`can't call openParent of a RemoteVhd before init`)
}
const parentPath = this.#vhd.header.parentUnicodeName
const fullParentPath = join(dirname(this.#path), parentPath)
if (!parentPath) {
throw new Error(`Disk ${this.#path} doesn't have parents`)
}
const parent = new RemoteVhd({ handler: this.#handler, path: fullParentPath })
return parent
}
/**
* Checks if the VHD is a differencing disk.
* @returns {boolean}
*/
isDifferencing() {
if (this.#isDifferencing === undefined) {
throw new Error(`can't call isDifferencing of a RemoteVhd before init`)
}
return this.#isDifferencing
}
}

View File

@@ -1,35 +0,0 @@
// @ts-check
/**
*
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
*/
import { DiskChain } from '@xen-orchestra/disk-transform'
import { RemoteVhd } from './RemoteVhd.mjs'
import { defer } from 'golike-defer'
/**
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {string} params.path
* @param {string | undefined} params.until
*/
async function _openDiskChain($defer, { handler, path, until }) {
let disk
const disks = []
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
disk = new RemoteVhd({ handler, path })
await disk.init()
disks.push(disk)
while (disk.isDifferencing()) {
disk = await disk.openParent()
if (disk.path === until) {
break
}
disks.unshift(disk)
}
// the root disk
return new DiskChain({ disks })
}
export const openDiskChain = defer(_openDiskChain)

View File

@@ -53,7 +53,7 @@ Add the following file: `xo-vm-backups/<VM UUID>/cache.json.gz`.
This cache is compressed in Gzip and contains an JSON object with the metadata for all the backups of this VM indexed by their absolute path (i.e. `/xo-vm-backups/<VM UUID>/<timestamp>.json`).
This file is generated on demand when listing the backups, and directly updated on backup creation/deletion.
This file is generated on demande when listing the backups, and directly updated on backup creation/deletion.
In case any incoherence is detected, the file is deleted so it will be fully generated when required.
@@ -180,7 +180,7 @@ job:
sourceRemote: IdPattern
# Indicates which XAPI to use to connect to a specific VM or SR
# for remote to remote backup job,this is only needed if there is healthcheck
# for remote to remote backup job,this is only needed if there is healtcheck
recordToXapi:
[ObjectId]: XapiId

View File

@@ -8,7 +8,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.63.0",
"version": "0.59.0",
"engines": {
"node": ">=14.18"
},
@@ -25,14 +25,11 @@
"@vates/decorate-with": "^2.1.0",
"@vates/disposable": "^0.1.6",
"@vates/fuse-vhd": "^2.1.2",
"@vates/generator-toolbox": "^1.0.4",
"@vates/nbd-client": "^3.1.4",
"@vates/nbd-client": "^3.1.2",
"@vates/parse-duration": "^0.1.1",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/disk-transform": "^1.1.0",
"@xen-orchestra/fs": "^4.6.2",
"@xen-orchestra/fs": "^4.5.0",
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/qcow2": "^1.0.0",
"@xen-orchestra/template": "^0.1.0",
"app-conf": "^3.0.0",
"compare-versions": "^6.0.0",
@@ -51,8 +48,8 @@
"tar": "^6.1.15",
"uuid": "^9.0.0",
"value-matcher": "^0.2.0",
"vhd-lib": "^4.14.0",
"xen-api": "^4.7.3",
"vhd-lib": "^4.11.3",
"xen-api": "^4.7.1",
"yazl": "^2.5.1"
},
"devDependencies": {
@@ -62,7 +59,7 @@
"tmp": "^0.2.1"
},
"peerDependencies": {
"@xen-orchestra/xapi": "^8.3.3"
"@xen-orchestra/xapi": "^8.1.1"
},
"license": "AGPL-3.0-or-later",
"author": {

View File

@@ -18,7 +18,7 @@
"preferGlobal": true,
"dependencies": {
"golike-defer": "^0.5.1",
"xen-api": "^4.7.3"
"xen-api": "^4.7.1"
},
"scripts": {
"postversion": "npm publish"

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,6 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended)

View File

@@ -1,40 +0,0 @@
{
"name": "@xen-orchestra/disk-transform",
"version": "1.1.0",
"main": "./dist/index.mjs",
"license": "MIT",
"private": false,
"type": "module",
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^5.0.1",
"@types/node": "^22.3.0",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0"
},
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "tsc && node --test **/*.test.mjs",
"clean": "rimraf dist/"
},
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/disk-transform",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/disk-transform",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"engines": {
"node": ">=20.17"
},
"dependencies": {
"vhd-lib": "^4.14.0",
"@vates/generator-toolbox": "^1.0.4"
}
}

View File

@@ -1,112 +0,0 @@
import { ProgressHandler } from './ProgressHandler.mjs'
export type DiskBlockData = Buffer
export type DiskBlock = {
index: number // the index of the block. Offset in raw disk is index * blockSize
data: DiskBlockData // thue Buffer like data of this block. Must be blockSize length
}
export type BytesLength = number
export abstract class Disk {
#generatedDiskBlocks = 0
#parent?: Disk
progressHandler?: ProgressHandler
get parent(): Disk | undefined {
return this.#parent
}
abstract getVirtualSize(): number
abstract getBlockSize(): number
abstract init(): Promise<void>
abstract close(): Promise<void>
abstract isDifferencing(): boolean
// optional method
instantiateParent(): Disk {
throw new Error('Method not implemented.')
}
async openParent(): Promise<Disk> {
if (this.#parent !== undefined) {
return this.#parent
}
if (!this.isDifferencing()) {
throw new Error("Can't open the parent of a non differencing disk")
}
this.#parent = this.instantiateParent()
await this.#parent.init()
return this.#parent
}
/**
* return the block without any order nor stability guarantee
*/
abstract getBlockIndexes(): Array<number>
// must return true if the block is present in this disk
// without looking at the parent
abstract hasBlock(index: number): boolean
abstract buildDiskBlockGenerator(): Promise<AsyncGenerator<DiskBlock>> | AsyncGenerator<DiskBlock>
async *diskBlocks(uid?: string): AsyncGenerator<DiskBlock> {
try {
// compute next block while the destination is consuming the current block
const blockGenerator = await this.buildDiskBlockGenerator()
let next = blockGenerator.next()
while (true) {
const res = await next
next = blockGenerator.next()
if (res.done) {
break
}
this.#generatedDiskBlocks++
yield res.value
}
} finally {
await this.close()
}
}
getNbGeneratedBlock(): number {
return this.#generatedDiskBlocks
}
check() {
// @todo move here the checks done in cleanVm if possible
return Promise.resolve(true)
}
// @todo readRawData will be needed for file level restore
// @todo rename will be needed for merge
// @todo : should the alias part moved from remote/vhd ?
}
/**
* Disks implementing this class can be accessed in any order
* For example xva will need to have data in the offset order
*/
export abstract class RandomAccessDisk extends Disk {
get parent(): RandomAccessDisk | undefined {
if (super.parent !== undefined && !(super.parent instanceof RandomAccessDisk)) {
throw new Error('The parent of a random access disk must be a random access disk')
}
return super.parent
}
// optional method
instantiateParent(): RandomAccessDisk {
throw new Error('Method not implemented.')
}
// can read data parent if block size are not aligned
// but only if this disk has data on this block
abstract readBlock(index: number): Promise<DiskBlock>
async *buildDiskBlockGenerator(): AsyncGenerator<DiskBlock> {
try {
const indexes = this.getBlockIndexes()
for (let i = 0; i < indexes.length; i++) {
yield this.readBlock(indexes[i])
await this.progressHandler?.setProgress(i / indexes.length)
}
await this.progressHandler?.setProgress(1)
} finally {
await this.progressHandler?.done()
}
}
}

View File

@@ -1,42 +0,0 @@
import { afterEach, beforeEach, test } from 'node:test'
import assert from 'node:assert'
import { DebugDisk } from './utils/DebugDisk.mjs'
test('Disk class', async t => {
const blockSize = 1024
const nbBlocks = 20
let disk: DebugDisk
beforeEach(async () => {
disk = new DebugDisk({ nbBlocks, blockSize, fillRate: 0 })
await disk.init()
})
afterEach(async () => {
await disk.close()
})
await t.test('getVirtualSize', () => {
assert.strictEqual(disk.getVirtualSize(), blockSize * nbBlocks)
})
await t.test('getBlockSize', () => {
assert.strictEqual(disk.getBlockSize(), blockSize)
})
await t.test('diskBlocks generator', async () => {
const keys = [4, 15]
for (const key of keys) {
disk.blocks[key] = true
}
const foundKeys = []
const generator = disk.diskBlocks()
for await (const block of generator) {
foundKeys.push(block.index)
assert.strictEqual(block.data.length, blockSize)
}
assert.strictEqual(disk.getNbGeneratedBlock(), 2)
keys.sort()
foundKeys.sort()
assert.deepStrictEqual(keys, foundKeys)
})
})

View File

@@ -1,69 +0,0 @@
import { Disk, RandomAccessDisk, type DiskBlock } from './Disk.mjs'
export class DiskChain extends RandomAccessDisk {
#disks: Array<RandomAccessDisk> = []
constructor({ disks }: { disks: Array<RandomAccessDisk> }) {
super()
this.#disks = disks
}
getVirtualSize(): number {
return this.#disks[this.#disks.length - 1].getVirtualSize()
}
getBlockSize(): number {
return this.#disks[0].getBlockSize()
}
/**
* the main difference with the base disk block method
* is that if any of the disk has this block it return true
*
*/
hasBlock(index: number): boolean {
for (let i = this.#disks.length - 1; i >= 0; i--) {
if (this.#disks[i].hasBlock(index)) {
return true
}
}
return false
}
/**
* Read in the last disk first an only go back if the block is not present
* @param index
* @returns
*/
readBlock(index: number): Promise<DiskBlock> {
for (let i = this.#disks.length - 1; i >= 0; i--) {
if (this.#disks[i].hasBlock(index)) {
return this.#disks[i].readBlock(index)
}
}
throw new Error(`Block ${index} not found in chain `)
}
async init(): Promise<void> {
await Promise.all(this.#disks.map(disk => disk.init()))
}
async close(): Promise<void> {
await Promise.all(this.#disks.map(disk => disk.close()))
}
getBlockIndexes(): Array<number> {
return [...new Set(this.#disks.map(disk => disk.getBlockIndexes()).flat())]
}
isDifferencing(): boolean {
return this.#disks[0].isDifferencing()
}
static async openFromChild(child:RandomAccessDisk):Promise<RandomAccessDisk>{
let disk = child
const disks=[disk]
while (disk.isDifferencing()) {
disk = await disk.openParent() as RandomAccessDisk
disks.unshift(disk)
// @todo handle until
}
return new DiskChain({ disks })
}
}

View File

@@ -1,62 +0,0 @@
import { afterEach, beforeEach, test } from 'node:test'
import assert from 'node:assert'
import { DebugDisk } from './utils/DebugDisk.mjs'
import { DiskChain } from './DiskChain.mjs'
test('Disk Chain class', async t => {
const blockSize = 1024
const nbBlocks = 20
const nbDisks = 5
const disks: Array<DebugDisk> = []
let chain: DiskChain
beforeEach(async () => {
for (let i = 0; i < nbDisks; i++) {
const disk = new DebugDisk({ nbBlocks: nbBlocks + 1, blockSize, fillRate: 0, filledBy: i })
disks.push(disk)
}
chain = new DiskChain({ disks })
await chain.init()
})
afterEach(async () => {
await chain.close()
})
await t.test('getVirtualSize', () => {
// the size of the last one
assert.strictEqual(chain.getVirtualSize(), disks[disks.length - 1].getVirtualSize())
})
await t.test('getBlockSize', () => {
assert.strictEqual(chain.getBlockSize(), blockSize)
})
await t.test('diskBlocks generator read from the right disk', async () => {
const keys = []
for (let blockIndex = 0; blockIndex < nbBlocks; blockIndex++) {
disks[0].blocks[blockIndex] = true //each disk only have one block, its disk index
keys.push(blockIndex)
}
for (let diskIndex = 0; diskIndex < nbDisks; diskIndex++) {
disks[diskIndex].blocks[diskIndex] = true //each disk only have one block, its disk index
}
const foundKeys = []
const generator = chain.diskBlocks()
for await (const block of generator) {
foundKeys.push(block.index)
assert.strictEqual(block.data.length, blockSize)
if (block.index < nbDisks) {
assert.strictEqual(block.data.readUInt8(0), block.index)
} else {
// read from first
assert.strictEqual(block.data.readUInt8(0), 0)
}
}
assert.strictEqual(chain.getNbGeneratedBlock(), nbBlocks)
keys.sort()
foundKeys.sort()
assert.deepStrictEqual(keys, foundKeys)
})
})

View File

@@ -1,106 +0,0 @@
import assert from 'node:assert'
import { DiskBlock, RandomAccessDisk } from './Disk.mjs'
import { RandomDiskPassthrough } from './DiskPassthrough.mjs'
import { DiskChain } from './DiskChain.mjs'
export class DiskLargerBlock extends RandomAccessDisk {
#blockSize
#source: RandomAccessDisk
#parent?: RandomAccessDisk
get source(): RandomAccessDisk {
return this.#source
}
constructor(source: RandomAccessDisk, blockSize: number) {
super()
this.#source = source
assert.ok(
blockSize >= source.getBlockSize(),
`target block size ${blockSize} must be bigger than the source block size ${source.getBlockSize()} `
)
assert.strictEqual(
blockSize % source.getBlockSize(),
0,
`target block size ${blockSize} must be a multiple of the source block size ${source.getBlockSize()} `
)
this.#blockSize = blockSize
}
openSource(): Promise<RandomAccessDisk> {
// not an issue since source MUST BE passed to the constructor
throw new Error('Method not implemented.')
}
async readBlock(index: number): Promise<DiskBlock> {
// @todo handle partial block at the end
const source = this.source
const destinationBlock = Buffer.alloc(this.getBlockSize(), 0)
const blockRatio = this.#blockSize / source.getBlockSize()
for (let i = index * blockRatio; i < (index + 1) * blockRatio; i++) {
let data: Buffer | undefined
if (source.hasBlock(i)) {
data = (await source.readBlock(i)).data
} else {
if (this.isDifferencing()) {
if (this.#parent === undefined) {
const directParent = (await source.openParent()) as RandomAccessDisk
const chain = await DiskChain.openFromChild(directParent)
this.#parent = chain
}
const parent = this.#parent!
data = (await parent.readBlock(i)).data
}
}
if (data !== undefined) {
data.copy(destinationBlock, i * source.getBlockSize())
}
}
return {
index,
data: destinationBlock,
}
}
getBlockSize(): number {
return this.#blockSize
}
getBlockIndexes(): Array<number> {
const maxBlock = Math.ceil(this.getVirtualSize() / this.getBlockSize())
const blockRatio = this.getBlockSize() / this.source.getBlockSize()
const indexes = []
for (let i = 0; i < maxBlock; i++) {
for (let j = 0; j < blockRatio; j++) {
if (this.source.hasBlock(i * blockRatio + j)) {
indexes.push(i)
break
}
}
}
return indexes
}
hasBlock(index: number): boolean {
const source = this.source
let maxBlockSource = Math.ceil(source.getVirtualSize() / source.getBlockSize())
let blockRatio = this.#blockSize / this.source.getBlockSize()
for (let i = index * blockRatio; i < Math.min((index + 1) * blockRatio, maxBlockSource); i++) {
if (this.source.hasBlock(i)) {
return true
}
}
return false
}
getVirtualSize(): number {
return this.source.getVirtualSize()
}
init(): Promise<void> {
return this.source.init()
}
close(): Promise<void> {
return this.source.close()
}
isDifferencing(): boolean {
return this.source.isDifferencing()
}
}

View File

@@ -1,227 +0,0 @@
import assert from 'node:assert'
import { test } from 'node:test'
import { DiskLargerBlock } from './DiskLargerBlock.mjs'
import { RandomAccessDisk, DiskBlock, Disk } from './Disk.mjs'
// Mock implementation of RandomAccessDisk for testing
class MockDisk extends RandomAccessDisk {
private blockSize: number
private virtualSize: number
private blocks: Map<number, Buffer>
public closed: boolean = false
public initialized: boolean = false
public isDiff: boolean = false
public parentDisk: RandomAccessDisk | null = null
constructor(blockSize: number, virtualSize: number, blocks: [number, Buffer][] = []) {
super()
this.blockSize = blockSize
this.virtualSize = virtualSize
this.blocks = new Map(blocks)
}
async readBlock(index: number): Promise<DiskBlock> {
if (this.closed) throw new Error('Disk closed')
if (!this.initialized) throw new Error('Disk not initialized')
if (!this.blocks.has(index)) {
if (this.isDiff && this.parentDisk) {
return this.parentDisk.readBlock(index)
}
throw new Error(`Block ${index} not found`)
}
return { index, data: this.blocks.get(index)! }
}
getVirtualSize(): number {
return this.virtualSize
}
getBlockSize(): number {
return this.blockSize
}
async init(): Promise<void> {
this.initialized = true
}
async close(): Promise<void> {
this.closed = true
}
isDifferencing(): boolean {
return this.isDiff
}
instantiateParent(): RandomAccessDisk {
if (!this.parentDisk) throw new Error('No parent disk available')
return this.parentDisk
}
getBlockIndexes(): number[] {
return [...this.blocks.keys()]
}
hasBlock(index: number): boolean {
return this.blocks.has(index) || (this.isDiff && !!this.parentDisk && this.parentDisk.hasBlock(index))
}
}
// Helper function to create a buffer with pattern
function createPatternBuffer(size: number, pattern: string): Buffer {
const buf = Buffer.alloc(size)
for (let i = 0; i < size; i++) {
buf[i] = pattern.charCodeAt(i % pattern.length)
}
return buf
}
// Test suite
test('constructor and initialization', async () => {
const source = new MockDisk(512, 1024 * 1024)
const disk = new DiskLargerBlock(source, 1024)
// Test that blockSize must be a multiple of source blockSize
await assert.rejects(
async () => {
const invalidDisk = new DiskLargerBlock(source, 513)
await invalidDisk.init()
},
{ message: /must be a multiple/ }
)
// Test that blockSize must be bigger than source blockSize
await assert.rejects(
async () => {
const invalidDisk = new DiskLargerBlock(source, 128)
await invalidDisk.init()
},
{ message: /bigger/ }
)
await disk.init()
assert.strictEqual(disk.getBlockSize(), 1024)
assert.strictEqual(disk.getVirtualSize(), 1024 * 1024)
})
test('readBlock with simple block mapping', async () => {
// Create source disk with two 512-byte blocks
const block1 = createPatternBuffer(512, 'A')
const block2 = createPatternBuffer(512, 'B')
const source = new MockDisk(512, 1024, [
[0, block1],
[1, block2],
])
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
// Read the combined block (should be block1 + block2)
const result = await disk.readBlock(0)
assert.strictEqual(result.index, 0)
assert.strictEqual(result.data.length, 1024)
// Verify the data is correctly combined
assert(result.data.subarray(0, 512).equals(block1))
assert(result.data.subarray(512).equals(block2))
})
test('hasBlock behavior', async () => {
const source = new MockDisk(512, 2048, [
[0, Buffer.alloc(512)],
[2, Buffer.alloc(512)],
])
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
// Block 0 in disk (1024) covers blocks 0-1 in source (512)
// Only block 0 exists in source, so hasBlock(0) should return true
assert.strictEqual(disk.hasBlock(0), true)
// Block 1 in disk covers blocks 2-3 in source
// Block 2 exists but 3 doesn't, hasBlock(1) should still return true
assert.strictEqual(disk.hasBlock(1), true)
// Block 2 in disk would cover blocks 4-5 which don't exist
assert.strictEqual(disk.hasBlock(2), false)
})
test('getBlockIndexes', async () => {
const source = new MockDisk(512, 2048, [
[0, Buffer.alloc(512)],
[2, Buffer.alloc(512)],
])
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
// Should return indexes where at least one source block exists in the range
const indexes = disk.getBlockIndexes()
assert.deepStrictEqual(indexes, [0, 1])
})
test('partial block handling', async () => {
// Test when the last block is partial
const source = new MockDisk(512, 1536, [
[0, Buffer.alloc(512)],
[1, Buffer.alloc(512)],
[2, Buffer.alloc(512)],
])
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
// First block is complete (blocks 0-1)
const block0 = await disk.readBlock(0)
assert.strictEqual(block0.data.length, 1024)
// Second block is partial (only block 2 exists)
const block1 = await disk.readBlock(1)
assert.strictEqual(block1.data.length, 1024)
// The first 512 bytes should be from block 2
// The remaining 512 bytes should be zeros (since we allocate with zeros)
assert(block1.data.subarray(512).every(b => b === 0))
assert(block1.data.subarray(0, 512).equals((await source.readBlock(2)).data))
})
test('differencing disk behavior', async () => {
// Create parent disk
const parentBlock = createPatternBuffer(512, 'PARENT')
const parent = new MockDisk(512, 1024, [[0, parentBlock]])
await parent.init()
// Create differencing disk
const diffBlock = createPatternBuffer(512, 'DIFF')
const diff = new MockDisk(512, 1024, [[1, diffBlock]])
diff.isDiff = true
diff.parentDisk = parent
await diff.init()
const disk = new DiskLargerBlock(diff, 1024)
await disk.init()
// Block 0 should combine parent block 0 and diff block 1
const result = await disk.readBlock(0)
assert.strictEqual(result.data.length, 1024)
assert(result.data.subarray(0, 512).equals(parentBlock))
assert(result.data.subarray(512).equals(diffBlock))
})
test('close propagation', async () => {
const source = new MockDisk(512, 1024)
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
await disk.close()
assert.strictEqual(source.closed, true)
})
test('generator', async () => {
const source = new MockDisk(512, 1024)
await source.init()
const disk = new DiskLargerBlock(source, 1024)
await disk.init()
for await (const block of disk.diskBlocks()) {
assert.strictEqual(block.data.length, 1024)
}
})

View File

@@ -1,108 +0,0 @@
import { Disk, DiskBlock, RandomAccessDisk } from './Disk.mjs'
export class DiskPassthrough extends Disk {
#source: Disk | undefined
get source(): Disk {
if (this.#source === undefined) {
throw new Error(`Either transmit the source to the constructor or implement openSource and call init`)
}
return this.#source
}
get parent(): Disk | undefined {
return this.#source?.parent
}
constructor(source: Disk | undefined = undefined) {
super()
this.#source = source
}
getVirtualSize(): number {
return this.source.getVirtualSize()
}
getBlockSize(): number {
return this.source.getBlockSize()
}
async openSource(): Promise<Disk> {
throw new Error('open source should be implemented to handle complex open scenario')
}
async init(): Promise<void> {
// open only if nothing has been given to the constructor
this.#source = this.#source ?? (await this.openSource())
}
instantiateParent(): Disk {
return this.source.instantiateParent()
}
async close(): Promise<void> {
await this.#source?.close()
}
isDifferencing(): boolean {
return this.source.isDifferencing()
}
getBlockIndexes(): Array<number> {
return this.source.getBlockIndexes()
}
hasBlock(index: number): boolean {
return this.source.hasBlock(index)
}
async buildDiskBlockGenerator(): Promise<AsyncGenerator<DiskBlock>> {
return this.source.buildDiskBlockGenerator()
}
}
export abstract class RandomDiskPassthrough extends RandomAccessDisk {
#source: RandomAccessDisk | undefined
get source(): RandomAccessDisk {
if (this.#source === undefined) {
throw new Error(`Either transmit the source to the constructor or implement openSource and call init`)
}
return this.#source
}
get parent(): RandomAccessDisk | undefined {
return this.#source?.parent as RandomAccessDisk
}
constructor(source: RandomAccessDisk | undefined) {
super()
this.#source = source
}
/**
* return an empty block if asking for and block not included in this disk
* @param index
* @returns {Promise<DiskBlock>}
*/
readBlock(index: number): Promise<DiskBlock> {
return this.source.readBlock(index)
}
getVirtualSize(): number {
return this.source.getVirtualSize()
}
getBlockSize(): number {
return this.source.getBlockSize()
}
abstract openSource(): Promise<RandomAccessDisk>
async init(): Promise<void> {
// open only if nothing has been given to the constructor
this.#source = this.#source ?? (await this.openSource())
}
instantiateParent(): RandomAccessDisk {
return this.source.instantiateParent()
}
async close(): Promise<void> {
await this.source?.close()
}
isDifferencing(): boolean {
return this.source.isDifferencing()
}
getBlockIndexes(): Array<number> {
return this.source.getBlockIndexes()
}
hasBlock(index: number): boolean {
return this.source.hasBlock(index)
}
}

View File

@@ -1,44 +0,0 @@
import { afterEach, beforeEach, test } from 'node:test'
import assert from 'node:assert'
import { DebugDisk } from './utils/DebugDisk.mjs'
import { DiskPassthrough } from './DiskPassthrough.mjs'
import { consume } from './utils/DebugConsumer.mjs'
test('Disk Passthrough', async t => {
const blockSize = 1024
const nbBlocks = 20
let disk: DebugDisk
beforeEach(async () => {
disk = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })
await disk.init()
})
afterEach(async () => {
await disk.close()
})
await t.test('source in constructor', async () => {
const passthrough = new DiskPassthrough(disk)
await assert.rejects(() => passthrough.openSource())
assert.strictEqual(passthrough.getBlockSize(), blockSize)
assert.strictEqual(passthrough.getVirtualSize(), blockSize * nbBlocks)
await consume(passthrough)
assert.strictEqual(passthrough.getNbGeneratedBlock(), nbBlocks)
assert.strictEqual(disk.closeDone, true)
})
await t.test('source open async', async () => {
class DebugPT extends DiskPassthrough {
async openSource() {
return Promise.resolve(disk)
}
}
const passthrough = new DebugPT()
assert.throws(() => passthrough.source)
await passthrough.init()
assert.strictEqual(passthrough.getBlockSize(), blockSize)
assert.strictEqual(passthrough.getVirtualSize(), blockSize * nbBlocks)
await consume(passthrough)
assert.strictEqual(passthrough.getNbGeneratedBlock(), nbBlocks)
assert.strictEqual(disk.closeDone, true)
})
})

View File

@@ -1,99 +0,0 @@
import assert, { throws } from 'node:assert'
import { RandomAccessDisk, type Disk, type DiskBlock } from './Disk.mjs'
import { DiskPassthrough } from './DiskPassthrough.mjs'
export class DiskSmallerBlock extends DiskPassthrough {
#blockSize
#generatedDiskBlocks = 0
#blockCache?: DiskBlock
constructor(source: Disk, blockSize: number) {
super(source)
assert.ok(
blockSize <= source.getBlockSize(),
`target block size ${blockSize} must be smaller than the source block size ${source.getBlockSize()} `
)
assert.strictEqual(
source.getBlockSize() % blockSize,
0,
`target block size ${blockSize} must be a multiple of the source block size ${source.getBlockSize()} `
)
this.#blockSize = blockSize
}
openSource(): Promise<Disk> {
// not an issue since source MUST BE passed to the constructor
throw new Error('Method not implemented.')
}
getNbGeneratedBlock(): number {
return this.#generatedDiskBlocks
}
async *diskBlocks(): AsyncGenerator<DiskBlock> {
try {
const blockRatio = this.source.getBlockSize() / this.#blockSize
const sourceGenerator = this.source.diskBlocks()
for await (const { data: sourceData, index: sourceIndex } of sourceGenerator) {
for (let i = 0; i < blockRatio; i++) {
this.#generatedDiskBlocks++
const data = sourceData.subarray(i * this.#blockSize, (i + 1) * this.#blockSize)
yield {
index: sourceIndex * blockRatio + i,
data,
}
}
}
} finally {
await this.progressHandler?.done()
await this.close()
}
}
getBlockSize(): number {
return this.#blockSize
}
getBlockIndexes(): Array<number> {
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
const sourceIndexes = this.source.getBlockIndexes()
const indexes: Array<number> = []
const maxIndex = Math.ceil(this.getVirtualSize() / this.getBlockSize())
for (const sourceIndex of sourceIndexes) {
for (let i = 0; i < blockRatio; i++) {
const index = sourceIndex * blockRatio + i
if (index < maxIndex) {
indexes.push(index)
}
}
}
return indexes
}
hasBlock(index: number): boolean {
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
const sourceIndex = Math.floor(index / blockRatio)
return this.source.hasBlock(sourceIndex)
}
async readBlock(index: number): Promise<DiskBlock> {
if (this.source instanceof RandomAccessDisk) {
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
const sourceBlockIndex = Math.floor(index / blockRatio)
const indexInblock = index % blockRatio
let block = this.#blockCache
if (block?.index !== sourceBlockIndex) {
this.#blockCache = block = await this.source.readBlock(sourceBlockIndex)
}
const data = block.data.subarray(indexInblock * this.#blockSize, (indexInblock + 1) * this.#blockSize)
return {
index,
data,
}
} else {
throw new Error('not implemented')
}
}
}

View File

@@ -1,184 +0,0 @@
import assert from 'node:assert'
import { test } from 'node:test'
import { DiskSmallerBlock } from './DiskSmallerBlock.mjs'
import { RandomAccessDisk, DiskBlock, Disk } from './Disk.mjs'
// Mock implementation of RandomAccessDisk for testing
class MockDisk extends RandomAccessDisk {
private blockSize: number
private virtualSize: number
private blocks: Map<number, Buffer>
public closed: boolean = false
public initialized: boolean = false
public isDiff: boolean = false
public parentDisk: RandomAccessDisk | null = null
constructor(blockSize: number, virtualSize: number, blocks: [number, Buffer][] = []) {
super()
this.blockSize = blockSize
this.virtualSize = virtualSize
this.blocks = new Map(blocks)
}
async readBlock(index: number): Promise<DiskBlock> {
if (this.closed) throw new Error('Disk closed')
if (!this.initialized) throw new Error('Disk not initialized')
if (!this.blocks.has(index)) {
if (this.isDiff && this.parentDisk) {
return this.parentDisk.readBlock(index)
}
throw new Error(`Block ${index} not found`)
}
return { index, data: this.blocks.get(index)! }
}
getVirtualSize(): number {
return this.virtualSize
}
getBlockSize(): number {
return this.blockSize
}
async init(): Promise<void> {
this.initialized = true
}
async close(): Promise<void> {
this.closed = true
}
isDifferencing(): boolean {
return this.isDiff
}
instantiateParent(): RandomAccessDisk {
if (!this.parentDisk) throw new Error('No parent disk available')
return this.parentDisk
}
getBlockIndexes(): number[] {
return [...this.blocks.keys()]
}
hasBlock(index: number): boolean {
return this.blocks.has(index) || (this.isDiff && !!this.parentDisk && this.parentDisk.hasBlock(index))
}
}
// Helper function to create a buffer with pattern
function createPatternBuffer(size: number, pattern: string): Buffer {
const buf = Buffer.alloc(size)
for (let i = 0; i < size; i++) {
buf[i] = pattern.charCodeAt(i % pattern.length)
}
return buf
}
// Test suite
test('constructor and initialization', async () => {
const source = new MockDisk(512, 1024 * 1024)
const disk = new DiskSmallerBlock(source, 256)
// Test that blockSize must be a multiple of source blockSize
await assert.rejects(
async () => {
const invalidDisk = new DiskSmallerBlock(source, 452)
await invalidDisk.init()
},
{ message: /must be a multiple/ }
)
// Test that blockSize must be smaller than source blockSize
await assert.rejects(
async () => {
const invalidDisk = new DiskSmallerBlock(source, 1024 * 2)
await invalidDisk.init()
},
{ message: /smaller/ }
)
await disk.init()
assert.strictEqual(disk.getBlockSize(), 256)
assert.strictEqual(disk.getVirtualSize(), 1024 * 1024)
})
test('readBlock with simple block mapping', async () => {
// Create source disk with two 512-byte blocks
const block1 = createPatternBuffer(512, 'A')
const block2 = createPatternBuffer(512, 'B')
const source = new MockDisk(512, 1024, [
[0, block1],
[1, block2],
])
await source.init()
const disk = new DiskSmallerBlock(source, 256)
await disk.init()
// Read the first block , half
const result = await disk.readBlock(2)
assert.strictEqual(result.index, 2)
assert.strictEqual(result.data.length, 256)
// Verify the data is correctly combined
assert(block2.subarray(0, 256).equals(result.data))
})
test('readBlock interleaved', async () => {
// Create source disk with two 512-byte blocks
const block1 = createPatternBuffer(512, 'A')
const block2 = createPatternBuffer(512, 'B')
const source = new MockDisk(512, 1024, [
[0, block1],
[1, block2],
])
await source.init()
const disk = new DiskSmallerBlock(source, 256)
await disk.init()
// Read the first block , half
await disk.readBlock(1)
// then second block
await disk.readBlock(2)
// then back to first
const result = await disk.readBlock(0)
// Verify the data is correctly combined
assert(block1.subarray(0, 256).equals(result.data))
})
test('hasBlock behavior', async () => {
const source = new MockDisk(512, 2048, [
[0, Buffer.alloc(512)],
[2, Buffer.alloc(512)],
])
await source.init()
const disk = new DiskSmallerBlock(source, 256)
await disk.init()
assert(disk.hasBlock(0))
assert(disk.hasBlock(1))
assert(!disk.hasBlock(2))
assert(!disk.hasBlock(3))
assert(disk.hasBlock(4))
assert(disk.hasBlock(5))
})
test('getBlockIndexes', async () => {
const source = new MockDisk(512, 2048, [
[0, Buffer.alloc(512)],
[2, Buffer.alloc(512)],
])
await source.init()
const disk = new DiskSmallerBlock(source, 256)
await disk.init()
const indexes = disk.getBlockIndexes()
assert.deepStrictEqual(indexes, [0, 1, 4, 5])
})
test('close propagation', async () => {
const source = new MockDisk(512, 1024)
await source.init()
const disk = new DiskSmallerBlock(source, 256)
await disk.init()
await disk.close()
assert(source.closed)
})

View File

@@ -1,26 +0,0 @@
import type { Readable } from 'node:stream'
export type FileDescriptor = number
/**
* a base interface is defined to be able to add other accessor ,
* for example in browser or from a vmware datastore
*
* NOTE : file accessor on datastore should try to use a stream
* and read/skip from this when possible, since opening a new stream is costly
* and we are generally reading the vhd sequentially
*/
export interface FileAccessor {
closeFile(filehandle: number): Promise<void>
openFile(path: string, opts?: unknown): Promise<number>
read(
file: string | number,
buffer: Buffer | DataView,
position: number
): Promise<{ bytesRead: number; buffer: Buffer | DataView }>
getSize(path: string): Promise<number>
mktree(path: string): Promise<void>
rmtree(path: string): Promise<void>
unlink(path: string): Promise<void>
outputStream(stream: Readable): Promise<void>
}

View File

@@ -1,55 +0,0 @@
import { DiskBlock, Disk, RandomAccessDisk } from './Disk.mjs'
import assert from 'node:assert'
/**
* expose a differencing disk that contains the difference
* between child and parent
*
* the chain parent - child - negative will contains exactly the
* same data as parent, this is used to rewind a disk to a previous
* state
*/
export class NegativeDisk extends RandomAccessDisk {
#parent: RandomAccessDisk
#child: RandomAccessDisk
constructor(parent: RandomAccessDisk, child: RandomAccessDisk) {
super()
this.#parent = parent
this.#child = child
// @todo : should also check chain
assert.strictEqual(parent.getBlockSize(), child.getBlockSize(), 'GEOMETRY_CHANGED')
assert.strictEqual(parent.getVirtualSize(), child.getVirtualSize(), 'GEOMETRY_CHANGED')
}
getVirtualSize(): number {
return this.#child.getVirtualSize()
}
getBlockSize(): number {
return this.#child.getBlockSize()
}
readBlock(index: number): Promise<DiskBlock> {
if (this.#parent.hasBlock(index)) {
return this.#parent.readBlock(index)
}
// a new block => return an empty one
return Promise.resolve({
index,
data: Buffer.alloc(this.getBlockSize(), 0),
})
}
async init(): Promise<void> {
await Promise.all([this.#parent.init(), this.#child.init()])
}
async close(): Promise<void> {
await Promise.all([this.#parent.close(), this.#child.close()])
}
isDifferencing(): boolean {
return true
}
getBlockIndexes(): Array<number> {
return this.#child.getBlockIndexes()
}
hasBlock(index: number): boolean {
return this.#child.hasBlock(index)
}
}

View File

@@ -1,4 +0,0 @@
export interface ProgressHandler {
setProgress(total: number): void | Promise<void>
done(): void | Promise<void>
}

View File

@@ -1,63 +0,0 @@
import { DiskBlock, RandomAccessDisk } from "./Disk.mjs";
import { FileAccessor } from "./FileAccessor.mjs";
export class RawDisk extends RandomAccessDisk{
#accessor
#path
#descriptor:number|undefined
#blockSize:number
#size: number|undefined
constructor(accessor: FileAccessor, path: string, blockSize:number){
super()
this.#accessor= accessor
this.#path = path
this.#blockSize = blockSize
}
async readBlock(index: number): Promise<DiskBlock> {
if(this.#descriptor === undefined){
throw new Error("Can't call readBlock before init");
}
const data = Buffer.alloc(this.getBlockSize(), 0)
await this.#accessor.read(this.#descriptor, data, index*this.getBlockSize())
return {
index,
data
}
}
getVirtualSize(): number {
if(this.#size === undefined){
throw new Error("Can't call getVirtualsize before init");
}
return this.#size
}
getBlockSize(): number {
return this.#blockSize
}
async init(): Promise<void> {
this.#descriptor = await this.#accessor.openFile(this.#path)
this.#size = await this.#accessor.getSize(this.#path)
}
async close(): Promise<void> {
this.#descriptor && await this.#accessor.closeFile(this.#descriptor)
this.#descriptor = undefined
this.#size = undefined
}
isDifferencing(): boolean {
return false
}
getBlockIndexes(): Array<number> {
const nbBlocks = Math.ceil(this.getVirtualSize()/ this.getBlockSize())
const index =[]
for(let i=0; i < nbBlocks; i ++){
index.push(i)
}
return index
}
hasBlock(index: number): boolean {
return index * this.getBlockSize() < this.getVirtualSize()
}
}

View File

@@ -1,50 +0,0 @@
import { DiskBlock, RandomAccessDisk } from './Disk.mjs'
import { RandomDiskPassthrough } from './DiskPassthrough.mjs'
export class ReadAhead extends RandomDiskPassthrough {
#maxNumber: number
#maxPercent: number
constructor(source: RandomAccessDisk, { maxNumber = 10, maxPercent = 10 } = {}) {
super(source)
this.#maxNumber = maxNumber
this.#maxPercent = maxPercent
}
openSource(): Promise<RandomAccessDisk> {
throw new Error('Method not implemented.')
}
buildDiskBlockGenerator(): AsyncGenerator<DiskBlock> {
const self = this
const blockIndexes = this.getBlockIndexes()
const maxNumber = this.#maxNumber
const maxPercent = this.#maxPercent
async function* generator(): AsyncGenerator<DiskBlock> {
const preloaded = []
// between 1 and 10 or at most 10% of the blocks
const PRELOAD_SIZE = Math.max(Math.min(Math.floor((blockIndexes.length * maxPercent) / 100), maxNumber), 1)
let counter = 0
try{
for (const index of blockIndexes) {
counter++
await self.progressHandler?.setProgress(counter/ blockIndexes.length)
if (preloaded.length < PRELOAD_SIZE) {
preloaded.push(self.source.readBlock(index))
}
if (preloaded.length === PRELOAD_SIZE) {
const next = (await preloaded.shift())!
yield next
}
}
while (preloaded.length > 0) {
const next = (await preloaded.shift())!
yield next
}
}finally{
await self.progressHandler?.done()
}
}
return generator()
}
}

View File

@@ -1,52 +0,0 @@
import { DiskPassthrough } from './DiskPassthrough.mjs'
import { Synchronized } from '@vates/generator-toolbox'
import { Disk, DiskBlock } from './Disk.mjs'
class ForkedDisk extends DiskPassthrough {
#generator: AsyncGenerator<DiskBlock, any, any>
#generatedDiskBlocks = 0
constructor(source: Disk, generator: AsyncGenerator<DiskBlock, any, any>) {
super(source)
this.#generator = generator
}
async openSource(): Promise<Disk> {
throw new ErrorEvent(' No need to open forked disk ')
}
async init(): Promise<void> {
/* source has already been open , */
}
async *diskBlocks(): AsyncGenerator<DiskBlock> {
try {
for await (const block of this.#generator) {
this.#generatedDiskBlocks++
yield block
}
} finally {
await this.progressHandler?.done()
await this.close()
}
}
getNbGeneratedBlock(): number {
return this.#generatedDiskBlocks
}
}
export class SynchronizedDisk {
#synchronized: Synchronized<DiskBlock, any, any> | undefined
#source: Disk
constructor(source: Disk) {
this.#source = source
}
fork(uid: string): ForkedDisk {
if (this.#synchronized === undefined) {
const generator = this.#source.diskBlocks()
this.#synchronized = new Synchronized(generator)
}
return new ForkedDisk(this.#source, this.#synchronized.fork(uid) as AsyncGenerator<DiskBlock, any, any>)
}
close() {
return this.#source.close()
}
}

View File

@@ -1,32 +0,0 @@
import { test } from 'node:test'
import assert from 'node:assert'
import { DebugDisk } from './utils/DebugDisk.mjs'
import { SynchronizedDisk } from './SynchronizedDisk.mjs'
async function consumeGenerator(source: AsyncGenerator) {
for await (const block of source) {
}
return 'finished'
}
test('Disk class', async t => {
const testRes = await t.test('synchronised fork disk ', async () => {
const blockSize = 1024
const nbBlocks = 20
const source = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })
const disk = new SynchronizedDisk(source)
await source.init()
assert.strictEqual(source.getVirtualSize(), blockSize * 20, 'size should be source size')
assert.strictEqual(source.getBlockSize(), blockSize, 'block size should be source size')
const forks = new Array()
for (let i = 0; i < 2; i++) {
forks.push(disk.fork('generator_' + i))
}
assert.strictEqual(source.getNbGeneratedBlock(), 0, 'nothing consumed')
await Promise.all(forks.map(fork => consumeGenerator(fork.diskBlocks())))
await disk.close()
assert.strictEqual(source.closeDone, true)
assert.strictEqual(source.getNbGeneratedBlock(), nbBlocks)
})
console.log({ testRes })
})

View File

@@ -1,11 +0,0 @@
export { Disk, DiskBlock, RandomAccessDisk } from './Disk.mjs'
export { DiskChain } from './DiskChain.mjs'
export { DiskLargerBlock } from './DiskLargerBlock.mjs'
export { DiskSmallerBlock } from './DiskSmallerBlock.mjs'
export { NegativeDisk } from './NegativeDisk.mjs'
export { FileAccessor } from './FileAccessor.mjs'
export { DiskPassthrough, RandomDiskPassthrough } from './DiskPassthrough.mjs'
export { RawDisk } from './RawDisk.mjs'
export { ReadAhead } from './ReadAhead.mjs'
export { SynchronizedDisk } from './SynchronizedDisk.mjs'
export { ProgressHandler } from './ProgressHandler.mjs'

View File

@@ -1,33 +0,0 @@
// @ts-expect-error async-each is not typed
import { asyncEach } from '@vates/async-each'
import { Disk, DiskBlock } from '../Disk.mjs'
import { DebugDisk } from './DebugDisk.mjs'
/**
*
* @param disk utility method that consume a disk and show the speed
*/
export async function consume(disk: Disk, { blockDelay = 20, concurrency = 8 } = {}) {
const start = Date.now()
let consumed = 0,
nbBlocks = 0
await asyncEach(
disk.diskBlocks(),
async (block: DiskBlock) => {
consumed += block.data.length
nbBlocks++
await new Promise(resolve => setTimeout(resolve, blockDelay))
},
{ concurrency }
)
const end = Date.now()
const duration = end - start
console.log('consume', {
duration, // in ms
nbBlocks,
consumed: Math.round(consumed / 1024 / 1024), // in MB
speed: Math.round((consumed * 1000) / duration / 1024 / 1024), // in MB/s
})
}

View File

@@ -1,67 +0,0 @@
import { Disk, DiskBlock, RandomAccessDisk } from '../Disk.mjs'
/**
* utility class that generate a random disk
*/
export class DebugDisk extends RandomAccessDisk {
#nbBlocks: number
#blockSize: number
#fillRate: number // 0 - 100
#filledBy: number // 0 255
blocks: Array<boolean> = []
closeDone = false
initDone = false
constructor({
nbBlocks,
blockSize,
fillRate,
filledBy = 0,
}: {
nbBlocks: number
blockSize: number
fillRate: number
filledBy?: number
}) {
super()
this.#nbBlocks = nbBlocks
this.#blockSize = blockSize
this.#fillRate = fillRate
this.#filledBy = filledBy
}
getVirtualSize(): number {
return this.#blockSize * this.#nbBlocks
}
getBlockSize(): number {
return this.#blockSize
}
async init(): Promise<void> {
for (let i = 0; i < this.#nbBlocks; i++) {
this.blocks.push(Math.random() <= this.#fillRate / 100)
}
this.initDone = true
}
async close(): Promise<void> {
this.closeDone = true
}
isDifferencing(): boolean {
return false
}
getBlockIndexes(): Array<number> {
const indexes = []
for (let i = 0; i < this.#nbBlocks; i++) {
if (this.blocks[i]) {
indexes.push(i)
}
}
return indexes
}
hasBlock(index: number): boolean {
return this.blocks[index]
}
async readBlock(index: number): Promise<DiskBlock> {
return {
index,
data: Buffer.alloc(this.getBlockSize(), this.#filledBy),
}
}
}

View File

@@ -1,15 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"module": "ESNext",
"moduleResolution": "bundler",
"target": "es2022",
"skipLibCheck": true,
"declaration": true // Generates `.d.ts` files for type safety
},
"include": ["./src/**/*"],
"exclude": ["./node_modules", "./dist"]
}

View File

@@ -14,7 +14,7 @@ ee.on('start', async function () {
// whatever
})
// similar to EventEmitter#emit() but returns a promise which resolves when all
// similar to EventEmmiter#emit() but returns a promise which resolves when all
// listeners have settled
await ee.emitAsync('start')

View File

@@ -32,7 +32,7 @@ ee.on('start', async function () {
// whatever
})
// similar to EventEmitter#emit() but returns a promise which resolves when all
// similar to EventEmmiter#emit() but returns a promise which resolves when all
// listeners have settled
await ee.emitAsync('start')

View File

@@ -1,96 +0,0 @@
## Testing Azure Blob Storage Using Azurite emulator
### Prerequisites
- A Linux-based VM with Node.js installed
### Steps
1. Install Azurite
```sh
npm install -g azurite
```
2. Run Azurite
```sh
azurite --blobHost 0.0.0.0 --silent --location c:\azurite --debug c:\azurite\debug.log
```
---
**Explanation of Flags:**
- `--blobHost 0.0.0.0` → Binds Azurite to all network interfaces making it accessible from external machines
- `--silent` → Suppresses unnecessary logs
- `--location c:\azurite` → Specifies the directory where Azurite will store data
- `--debug c:\azurite\debug.log` → Enables debug logging for troubleshooting
---
3. Configure Connection String on Your Local Machine
Replace `AZURITE_CONNECTION_STRING` with the following connection string in the storage operations commands:
`DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://<VM_IP>:10000/devstoreaccount1;QueueEndpoint=http://<VM_IP>:10001/devstoreaccount1;TableEndpoint=http://<VM_IP>:10002/devstoreaccount1;`
**Replace `<VM_IP>` with the actual IP address of your VM where Azurite is running.**
**The account name (devstoreaccount1) and key provided in the connection string above are the default credentials for Azurite. These credentials are publicly documented and can be used by anyone for local development and testing**
---
### 4. Perform Storage Operations
Create a Container (Equivalent to an S3 Bucket)
```sh
az storage container create --name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
```
Upload a Blob (Equivalent to a File/Directory in S3)
```sh
az storage blob upload --account-name devstoreaccount1 --container-name <container_name> --name <blob_name> --file <file_path> --connection-string "AZURITE_CONNECTION_STRING"
```
List All Containers
```sh
az storage container list --connection-string "AZURITE_CONNECTION_STRING"
```
Download a Blob
```sh
az storage blob download --container-name <container_name> --name <blob_name> --file <file_name> --connection-string "AZURITE_CONNECTION_STRING"
```
Delete a Blob
```sh
az storage blob delete --container-name <container_name> --name <blob_name> --connection-string "AZURITE_CONNECTION_STRING"
```
List Blobs Inside a Container
```sh
az storage blob list --container-name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
```
Delete a Container
```sh
az storage container delete --name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
```
---
### Notes
- Ensure your VM allows incoming connections on ports `10000`, `10001`, and `10002` for the Blob, Queue, and Table endpoints, respectively
- Use `--debug` flag while running commands if you need detailed logs
- Replace placeholders (`<VM_IP>`, `<container_name>`, `<blob_name>`, `<file_path>`, `<file_name>`) with actual values before executing commands
Source documentation: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=npm%2Cblob-storage

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/fs",
"version": "4.6.2",
"version": "4.5.0",
"license": "AGPL-3.0-or-later",
"description": "The File System for Xen Orchestra backups.",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -25,7 +25,6 @@
"@aws-sdk/lib-storage": "^3.54.0",
"@aws-sdk/middleware-apply-body-checksum": "^3.58.0",
"@aws-sdk/node-http-handler": "^3.54.0",
"@azure/storage-blob": "^12.26.0",
"@vates/async-each": "^1.0.0",
"@vates/coalesce-calls": "^0.1.0",
"@vates/decorate-with": "^2.1.0",
@@ -42,7 +41,7 @@
"proper-lockfile": "^4.1.2",
"readable-stream": "^4.1.0",
"through2": "^4.0.2",
"xo-remote-parser": "^0.10.0"
"xo-remote-parser": "^0.9.3"
},
"devDependencies": {
"@babel/cli": "^7.0.0",

View File

@@ -3,14 +3,13 @@ import getStream from 'get-stream'
import { asyncEach } from '@vates/async-each'
import { coalesceCalls } from '@vates/coalesce-calls'
import { createLogger } from '@xen-orchestra/log'
import { fromCallback, fromEvent, ignoreErrors, pRetry } from 'promise-toolbox'
import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
import { limitConcurrency } from 'limit-concurrency-decorator'
import { parse } from 'xo-remote-parser'
import { pipeline } from 'stream'
import { randomBytes, randomUUID } from 'crypto'
import { synchronized } from 'decorator-synchronized'
import { withTimeout } from './utils'
import { basename, dirname, normalize as normalizePath } from './path'
import { createChecksumStream, validChecksumOfReadStream } from './checksum'
import { DEFAULT_ENCRYPTION_ALGORITHM, UNENCRYPTED_ALGORITHM, _getEncryptor } from './_encryptor'
@@ -29,58 +28,6 @@ const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
const ENCRYPTION_DESC_FILENAME = 'encryption.json'
const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
const WITH_LIMIT = [
'closeFile',
'copy',
'getInfo',
'getSizeOnDisk',
'list',
'mkdir',
'openFile',
'outputFile',
'read',
'readFile',
'rename',
'rmdir',
'truncate',
'unlink',
'write',
'writeFile',
]
const WITH_RETRY = [
'_closeFile',
'_copy',
'_getInfo',
'_getSize',
'_list',
'_mkdir',
'_openFile',
'_outputFile',
'_read',
'_readFile',
'_rename',
'_rmdir',
'_truncate',
'_unlink',
'_write',
'_writeFile',
]
const WITH_TIMEOUT = [
'_closeFile',
'_createReadStream',
'_getInfo',
'_getSize',
'_list',
'_rename',
'_copy',
'_rmdir',
'_closeFile',
'_openFile',
'_createOutputStream',
]
const ignoreEnoent = error => {
if (error == null || error.code !== 'ENOENT') {
throw error
@@ -146,59 +93,28 @@ export default class RemoteHandlerAbstract {
throw new Error('Incorrect remote type')
}
}
;({
highWaterMark: this._highWaterMark,
timeout: this._timeout = DEFAULT_TIMEOUT,
withLimit: this._withLimit = WITH_LIMIT,
withTimeout: this._withTimeout = WITH_TIMEOUT,
withRetry: this._withRetry = WITH_RETRY,
} = options)
;({ highWaterMark: this._highWaterMark, timeout: this._timeout = DEFAULT_TIMEOUT } = options)
this.#applySafeGuards(options)
}
#conditionRetry(error) {
return !['EEXIST', 'EISDIR', 'ENOTEMPTY', 'ENOENT', 'ENOTDIR', 'SystemInUse', 'ERR_ASSERTION'].includes(error?.code)
}
#applySafeGuards(options) {
const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
this.closeFile = sharedLimit(this.closeFile)
this.copy = sharedLimit(this.copy)
this.getInfo = sharedLimit(this.getInfo)
this.getSizeOnDisk = sharedLimit(this.getSizeOnDisk)
this.list = sharedLimit(this.list)
this.mkdir = sharedLimit(this.mkdir)
this.openFile = sharedLimit(this.openFile)
this.outputFile = sharedLimit(this.outputFile)
this.read = sharedLimit(this.read)
this.readFile = sharedLimit(this.readFile)
this.rename = sharedLimit(this.rename)
this.rmdir = sharedLimit(this.rmdir)
this.truncate = sharedLimit(this.truncate)
this.unlink = sharedLimit(this.unlink)
this.write = sharedLimit(this.write)
this.writeFile = sharedLimit(this.writeFile)
this._forget = coalesceCalls(this._forget)
this._sync = coalesceCalls(this._sync)
this._withLimit.forEach(functionName => {
if (this[functionName] !== undefined) {
this[functionName] = sharedLimit(this[functionName])
}
})
this._withTimeout.forEach(functionName => {
if (this[functionName] !== undefined) {
this[functionName] = withTimeout(this[functionName], DEFAULT_TIMEOUT)
}
})
this._withRetry.forEach(functionName => {
if (this[functionName] !== undefined) {
// adding the retry on the top level method won't
// cover when _functionName are called internally
this[functionName] = pRetry.wrap(this[functionName], {
delays: [100, 200, 500, 1000, 2000],
// these errors should not change on retry
when: err => this.#conditionRetry(err),
onRetry(error) {
warn('retrying method on fs ', {
method: functionName,
attemptNumber: this.attemptNumber,
delay: this.delay,
error,
file: this.arguments?.[0],
})
},
})
}
})
}
// Public members
@@ -224,7 +140,11 @@ export default class RemoteHandlerAbstract {
file = normalizePath(file)
}
let stream = await this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark })
let stream = await timeout.call(
this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }),
this._timeout
)
// detect early errors
await fromEvent(stream, 'readable')
@@ -245,18 +165,14 @@ export default class RemoteHandlerAbstract {
if (this.isEncrypted) {
stream = this.#encryptor.decryptStream(stream)
}
// try to add the length prop if missing and not a range stream
if (stream.length === undefined && options.end === undefined && options.start === undefined) {
try {
if (this.isEncrypted) {
stream.maxStreamLength = await this.getSizeOnDisk(file)
} else {
} else {
// try to add the length prop if missing and not a range stream
if (stream.length === undefined && options.end === undefined && options.start === undefined) {
try {
stream.length = await this._getSize(file)
} catch (error) {
// ignore errors
}
} catch (error) {
// ignore errors
}
}
@@ -290,16 +206,16 @@ export default class RemoteHandlerAbstract {
validator,
})
if (checksum) {
// using _outputFile means the checksum will NOT be encrypted
// using _outpuFile means the checksum will NOT be encrypted
// it is by design to allow checking of encrypted files without the key
await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' })
}
}
// Free the resources possibly dedicated to put the remote at work, when it
// is no longer needed
// is no more needed
//
// FIXME: Some handlers are implemented based on system-wide mechanisms (such
// FIXME: Some handlers are implemented based on system-wide mecanisms (such
// as mount), forgetting them might breaking other processes using the same
// remote.
@synchronized()
@@ -308,7 +224,7 @@ export default class RemoteHandlerAbstract {
}
async getInfo() {
return this._getInfo()
return timeout.call(this._getInfo(), this._timeout)
}
// returns the real size occupied by an unencrypted file
@@ -319,7 +235,7 @@ export default class RemoteHandlerAbstract {
}
async getSizeOnDisk(file) {
return this._getSize(typeof file === 'string' ? normalizePath(file) : file)
return timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
}
async __list(dir, { filter, ignoreMissing = false, prependDir = false } = {}) {
@@ -327,7 +243,7 @@ export default class RemoteHandlerAbstract {
const virtualDir = normalizePath(dir)
dir = normalizePath(dir)
let entries = await this._list(dir)
let entries = await timeout.call(this._list(dir), this._timeout)
if (filter !== undefined) {
entries = entries.filter(filter)
}
@@ -373,7 +289,7 @@ export default class RemoteHandlerAbstract {
async #rename(oldPath, newPath, { checksum }, createTree = true) {
try {
let p = this._rename(oldPath, newPath)
let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
if (checksum) {
p = Promise.all([p, this._rename(checksumFile(oldPath), checksumFile(newPath))])
}
@@ -396,7 +312,7 @@ export default class RemoteHandlerAbstract {
oldPath = normalizePath(oldPath)
newPath = normalizePath(newPath)
let p = this._copy(oldPath, newPath)
let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
if (checksum) {
p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
}
@@ -404,14 +320,14 @@ export default class RemoteHandlerAbstract {
}
async rmdir(dir) {
await this._rmdir(normalizePath(dir)).catch(ignoreEnoent)
await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
}
async rmtree(dir) {
await this._rmtree(normalizePath(dir))
}
// Asks the handler to sync the state of the effective remote with its
// Asks the handler to sync the state of the effective remote with its'
// metadata
//
// This method MUST ALWAYS be called before using the handler.
@@ -466,7 +382,7 @@ export default class RemoteHandlerAbstract {
const data = await this.__readFile(ENCRYPTION_METADATA_FILENAME)
JSON.parse(data)
} catch (error) {
// can be enoent, bad algorithm, or broken json ( bad key or algorithm)
// can be enoent, bad algorithm, or broeken json ( bad key or algorithm)
if (encryptionAlgorithm !== 'none') {
if (await this.#canWriteMetadata()) {
// any other error , but on empty remote => update with remote settings
@@ -474,7 +390,7 @@ export default class RemoteHandlerAbstract {
info('will update metadata of this remote')
return this.#createMetadata()
} else {
// to add a new encrypted fs remote, the remote directory must be empty; otherwise, metadata.json is not created
// to add a new encrypted fs remote, the remote directory must be empty, otherwise metadata.json is not created
if (error.code === 'ENOENT' && error.path.includes('metadata.json')) {
throw new Error('Remote directory must be empty.')
}
@@ -553,7 +469,7 @@ export default class RemoteHandlerAbstract {
// Methods that can be called by private methods to avoid parallel limit on public methods
async __closeFile(fd) {
await this._closeFile(fd.fd)
await timeout.call(this._closeFile(fd.fd), this._timeout)
}
async __mkdir(dir, { mode } = {}) {
@@ -575,7 +491,7 @@ export default class RemoteHandlerAbstract {
path = normalizePath(path)
return {
fd: await this._openFile(path, flags),
fd: await timeout.call(this._openFile(path, flags), this._timeout),
path,
}
}
@@ -667,11 +583,14 @@ export default class RemoteHandlerAbstract {
}
async _outputStream(path, input, { dirMode, validator }) {
const tmpPath = `${dirname(path)}/.${Date.now()}.${basename(path)}`
const output = await this._createOutputStream(tmpPath, {
dirMode,
flags: 'wx',
})
const tmpPath = `${dirname(path)}/.${basename(path)}`
const output = await timeout.call(
this._createOutputStream(tmpPath, {
dirMode,
flags: 'wx',
}),
this._timeout
)
try {
await fromCallback(pipeline, input, output)
if (validator !== undefined) {
@@ -713,25 +632,25 @@ export default class RemoteHandlerAbstract {
}
const files = await this._list(dir)
await asyncEach(
files,
file =>
this._unlink(`${dir}/${file}`).catch(error => {
await asyncEach(files, file =>
this._unlink(`${dir}/${file}`).catch(
error => {
// Unlink dir behavior is not consistent across platforms
// https://github.com/nodejs/node-v0.x-archive/issues/5791
if (error.code === 'EISDIR' || error.code === 'EPERM') {
return this._rmtree(`${dir}/${file}`).catch(rmTreeError => {
if (rmTreeError.code === 'ENOTDIR') {
// this was really a EPERM error, maybe with immutable backups
// this was realy a EPERM error, maybe with immutable backups
throw error
}
throw rmTreeError
})
}
throw error
}),
// real unlink concurrency will be 2**max directory depth
{ concurrency: 2 }
},
// real unlink concurrency will be 2**max directory depth
{ concurrency: 2 }
)
)
return this._rmtree(dir)
}

View File

@@ -10,37 +10,17 @@ import AbstractHandler from './abstract'
import fs from 'fs-extra'
import tmp from 'tmp'
const TIMEOUT = 6e5
const TIMEOUT = 10e3
class TestHandler extends AbstractHandler {
constructor() {
const options = { timeout: TIMEOUT, withRetry: [] }
super({ url: 'test://' }, options)
constructor(impl) {
super({ url: 'test://' }, { timeout: TIMEOUT })
Object.defineProperty(this, 'isEncrypted', {
get: () => false, // encryption is tested separately
get: () => false, // encryption is tested separatly
})
Object.keys(impl).forEach(method => {
this[`_${method}`] = impl[method]
})
}
async _closeFile() {
Promise(() => {})
}
async _getInfo() {
Promise(() => {})
}
async _getSize() {
Promise(() => {})
}
async _list() {
Promise(() => {})
}
async _openFile() {
Promise(() => {})
}
async _rename() {
Promise(() => {})
}
async _rmdir() {
Promise(() => {})
}
}
@@ -50,7 +30,9 @@ const clock = sinon.useFakeTimers()
describe('closeFile()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
closeFile: () => new Promise(() => {}),
})
const promise = testHandler.closeFile({ fd: undefined, path: '' })
clock.tick(TIMEOUT)
@@ -60,7 +42,9 @@ describe('closeFile()', () => {
describe('getInfo()', () => {
it('throws in case of timeout', async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
getInfo: () => new Promise(() => {}),
})
const promise = testHandler.getInfo()
clock.tick(TIMEOUT)
@@ -70,7 +54,9 @@ describe('getInfo()', () => {
describe('getSize()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
getSize: () => new Promise(() => {}),
})
const promise = testHandler.getSize('')
clock.tick(TIMEOUT)
@@ -80,7 +66,9 @@ describe('getSize()', () => {
describe('list()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
list: () => new Promise(() => {}),
})
const promise = testHandler.list('.')
clock.tick(TIMEOUT)
@@ -90,7 +78,9 @@ describe('list()', () => {
describe('openFile()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
openFile: () => new Promise(() => {}),
})
const promise = testHandler.openFile('path')
clock.tick(TIMEOUT)
@@ -100,7 +90,9 @@ describe('openFile()', () => {
describe('rename()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
rename: () => new Promise(() => {}),
})
const promise = testHandler.rename('oldPath', 'newPath')
clock.tick(TIMEOUT)
@@ -110,7 +102,9 @@ describe('rename()', () => {
describe('rmdir()', () => {
it(`throws in case of timeout`, async () => {
const testHandler = new TestHandler()
const testHandler = new TestHandler({
rmdir: () => new Promise(() => {}),
})
const promise = testHandler.rmdir('dir')
clock.tick(TIMEOUT)

Some files were not shown because too many files have changed in this diff Show More