mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-16 03:51:39 -06:00
Compare commits
5 Commits
docs-fixes
...
create-tas
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d62433148b | ||
|
|
d44364d4d5 | ||
|
|
9118dc0835 | ||
|
|
97bf035a9f | ||
|
|
1e7ac62c51 |
12
.eslintrc.js
12
.eslintrc.js
@@ -171,6 +171,7 @@ module.exports = {
|
||||
'vue/no-empty-component-block': 'error',
|
||||
'vue/no-multiple-objects-in-class': 'error',
|
||||
'vue/no-ref-object-reactivity-loss': 'error',
|
||||
'vue/no-required-prop-with-default': 'error',
|
||||
'vue/no-static-inline-styles': 'error',
|
||||
'vue/no-template-target-blank': 'error',
|
||||
'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
|
||||
@@ -216,16 +217,7 @@ module.exports = {
|
||||
'@intlify/vue-i18n/no-raw-text': [
|
||||
'error',
|
||||
{
|
||||
ignoreText: [
|
||||
'Xen Orchestra Lite',
|
||||
'Xen Orchestra',
|
||||
'Xen Orchestra Appliance',
|
||||
'XO Lite',
|
||||
'XCP-ng',
|
||||
'XO 5',
|
||||
'404',
|
||||
'⚠️',
|
||||
],
|
||||
ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
|
||||
},
|
||||
],
|
||||
'@intlify/vue-i18n/no-v-html': 'error',
|
||||
|
||||
4
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
4
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@@ -40,8 +40,8 @@ body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
- Don't forget to [read this first](https://docs.xen-orchestra.com/community)
|
||||
- As well as follow [this guide](https://docs.xen-orchestra.com/community#report-a-bug)
|
||||
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
|
||||
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
|
||||
- type: input
|
||||
id: xo-sources-commit-number
|
||||
attributes:
|
||||
|
||||
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@@ -1,4 +1,4 @@
|
||||
name: Continuous Integration
|
||||
name: Continous Integration
|
||||
on: push
|
||||
jobs:
|
||||
CI:
|
||||
|
||||
@@ -16,7 +16,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
|
||||
|
||||
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
|
||||
- `signal`: an abort signal to stop the iteration
|
||||
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
|
||||
```js
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
@@ -34,7 +34,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
|
||||
|
||||
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
|
||||
- `signal`: an abort signal to stop the iteration
|
||||
- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
|
||||
```js
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
|
||||
* @param {T} data2 - The second data array or buffer to compare.
|
||||
* @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
|
||||
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided; otherwise, undefined.
|
||||
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
|
||||
*/
|
||||
module.exports = function diff(data1, data2, cb) {
|
||||
let result
|
||||
|
||||
@@ -14,7 +14,7 @@ import { deduped } from '@vates/disposable/deduped'
|
||||
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
|
||||
const getConnection = deduped(async function (host)) {
|
||||
const connection = new Connection(host)
|
||||
return new Disposable(connection, () => connection.close())
|
||||
return new Disposabe(connection, () => connection.close())
|
||||
}, host => [host])
|
||||
```
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ import { deduped } from '@vates/disposable/deduped'
|
||||
// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
|
||||
const getConnection = deduped(async function (host)) {
|
||||
const connection = new Connection(host)
|
||||
return new Disposable(connection, () => connection.close())
|
||||
return new Disposabe(connection, () => connection.close())
|
||||
}, host => [host])
|
||||
```
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ fs.stat('autoexec.bat', function (e, stats) {
|
||||
|
||||
## API
|
||||
|
||||
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back an instance of an [`fs`](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
|
||||
- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back the API documented [here](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
|
||||
- `'ready'` event — fired on `fs` when initial volume information has been determined and the API is ready to use. It is safe to call other `fs` methods before this fires **only if** you are sure the first sector will be readable and represents a valid FAT volume.
|
||||
- `'error'` event — fired if initialization fails for whatever reason.
|
||||
|
||||
@@ -36,7 +36,7 @@ fs.stat('autoexec.bat', function (e, stats) {
|
||||
|
||||
The `opts` dictionary you pass to `fatfs.createFileSystem` can contain any of the following options:
|
||||
|
||||
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overridden to `true`.
|
||||
- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overriden to `true`.
|
||||
- `noatime` — The FAT filesystem can track the last access time (just a date, actually) but this means every read operation would also incur some write overhead. Defaults to `true`, meaning by default access times will **not** be updated on reads. Set this to `false` to track access times.
|
||||
- `modmode` — chooses how `fs.chmod` (and the mode field from `fs.stat`–family calls) should map FAT attributes to POSIX permissions. Set to the number `0111` to map the readonly flag to the user's write bit being unset, and the archive/system/hidden flags to the user/group/other executable bits respectively. Set to the number `07000` to map the readonly flag to _all_ write bits being unset, and the archive/system/hidden flags to the sticky/setgid/setuid bits respectively. Set to `null` for readonly mapping. Defaults to `0111`.
|
||||
- `umask` — any bits _set_ in this octal number will be _unset_ in the 'mode' field from `fs.stat`–family calls. It does not affect anything else. Defaults to `process.umask()`, or `0022` if that is unavailable.
|
||||
@@ -53,7 +53,7 @@ Well, sort of…
|
||||
|
||||
### Temporary
|
||||
|
||||
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please, please, please, **make sure you have a backup** of any important drive/image/card you unleash this upon.
|
||||
- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please please please **make sure you have a backup** of any important drive/image/card you unleash this upon.
|
||||
- A few methods are not quite implemented, either: `fs.rename`, `fs.unlink` and `fs.rmdir`, as well as `fs.watchFile`/`fs.unwatchFile` and `fs.watch`. These are Coming Soon™.
|
||||
- There are several internal housekeeping items (redundant FAT tables, extra FAT32 information, etc.) that are not done. These do not seem to affect interop, but you may see warnings when repairing a filesystem written by this module.
|
||||
- Oh, and not to scare you, but if an IO error happens while writing, the library usually just bails — bubbling an error up to your callback as if it were a hot potato. Although some attempt has been made to do separate writes in the safest order (e.g. allocating an additional file cluster, then appending data into it, and then finally updating the file's size), but this behavior has not been thoroughly audited for all operations. There's certainly no attempt to retry/cleanup/rollback if a multi-step change runs into trouble partway through.
|
||||
|
||||
@@ -29,7 +29,7 @@ The `remoteUrl` can be found using the following command from the XOA CLI, and t
|
||||
|
||||
`xo-server-db ls remote`
|
||||
|
||||
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
|
||||
The `vhdPathInRemote` follows a file structure described [here](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
|
||||
|
||||
## Restore a file from a VHD using `fuse-vhd` CLI
|
||||
|
||||
@@ -42,7 +42,7 @@ The `vhdPathInRemote` follows a file structure described in [VM backups](https:/
|
||||
|
||||
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
|
||||
|
||||
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
|
||||
4. Mount the disk. Depending on the partition type, you may need to add some additionnal options. The `norecovery` option is used for ext3/ext4/xfs file systems, otherwise remove this option.
|
||||
|
||||
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ The `remoteUrl` can be found using the following command from the XOA CLI, and t
|
||||
|
||||
`xo-server-db ls remote`
|
||||
|
||||
The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
|
||||
The `vhdPathInRemote` follows a file structure described [here](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
|
||||
|
||||
## Restore a file from a VHD using `fuse-vhd` CLI
|
||||
|
||||
@@ -58,7 +58,7 @@ The `vhdPathInRemote` follows a file structure described in [VM backups](https:/
|
||||
|
||||
`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
|
||||
|
||||
4. Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
|
||||
4. Mount the disk. Depending on the partition type, you may need to add some additionnal options. The `norecovery` option is used for ext3/ext4/xfs file systems, otherwise remove this option.
|
||||
|
||||
`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512> --source=<mountedVhdPath> --target=<diskMountPoint>`
|
||||
|
||||
|
||||
@@ -19,12 +19,12 @@
|
||||
},
|
||||
"main": "./index.mjs",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/fs": "^4.6.2",
|
||||
"@xen-orchestra/fs": "^4.5.0",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"fuse-native": "^2.2.6",
|
||||
"lru-cache": "^7.14.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"vhd-lib": "^4.14.0"
|
||||
"vhd-lib": "^4.11.3"
|
||||
},
|
||||
"bin": {
|
||||
"xo-fuse-vhd": "./cli.mjs"
|
||||
|
||||
@@ -30,14 +30,14 @@ const wrappedGenerator = new Throttle(sourceGenerator, speed)
|
||||
|
||||
### Synchronized
|
||||
|
||||
Fork a generator. The rules are:
|
||||
Fork a generator. The rules ares:
|
||||
|
||||
- if the source returns, all the forks return
|
||||
- if the forks error, all the forks error with the same error
|
||||
- if a fork returns, it is stopped, but the generator continue with the other
|
||||
- if a fork errors, it is stopped, but the generator continue with the other
|
||||
- if all the forks return, the source is stopped
|
||||
- if all the forks error, the source is errored with the last error
|
||||
- if the source returns, all the forks returns
|
||||
- if the forks errors, all the forks errors with the same error
|
||||
- if a the fork return , it is stopped, but the generator continue with the other
|
||||
- if a the fork error , it is stopped, but the generator continue with the other
|
||||
- if all the fork return , the source is stopped
|
||||
- if all the fork error , the source is errored with the last error
|
||||
- the source start producing a packet when the fastest forked ask for it
|
||||
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
|
||||
|
||||
|
||||
@@ -46,14 +46,14 @@ const wrappedGenerator = new Throttle(sourceGenerator, speed)
|
||||
|
||||
### Synchronized
|
||||
|
||||
Fork a generator. The rules are:
|
||||
Fork a generator. The rules ares:
|
||||
|
||||
- if the source returns, all the forks return
|
||||
- if the forks error, all the forks error with the same error
|
||||
- if a fork returns, it is stopped, but the generator continue with the other
|
||||
- if a fork errors, it is stopped, but the generator continue with the other
|
||||
- if all the forks return, the source is stopped
|
||||
- if all the forks error, the source is errored with the last error
|
||||
- if the source returns, all the forks returns
|
||||
- if the forks errors, all the forks errors with the same error
|
||||
- if a the fork return , it is stopped, but the generator continue with the other
|
||||
- if a the fork error , it is stopped, but the generator continue with the other
|
||||
- if all the fork return , the source is stopped
|
||||
- if all the fork error , the source is errored with the last error
|
||||
- the source start producing a packet when the fastest forked ask for it
|
||||
- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
|
||||
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
{
|
||||
"name": "@vates/generator-toolbox",
|
||||
"version": "1.0.4",
|
||||
"version": "1.0.1",
|
||||
"main": "dist/index.mjs",
|
||||
"license": "MIT",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"rimraf": "^5.0.1",
|
||||
"@types/node": "^20.6",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.23.0"
|
||||
@@ -16,10 +15,7 @@
|
||||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"postversion": "npm publish --access public",
|
||||
"prebuild": "rimraf dist/",
|
||||
"predev": "yarn run prebuild",
|
||||
"test": "tsc && node --test **/*.test.mjs",
|
||||
"clean": "rimraf dist/"
|
||||
"test": "tsc && node --test **/*.test.mjs"
|
||||
},
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/generator-toolbox",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
|
||||
@@ -17,10 +17,8 @@ export class Synchronized<T, TReturn, TNext> {
|
||||
this.#source = source
|
||||
}
|
||||
|
||||
fork(uid: string): AsyncGenerator<T, TReturn, TNext> {
|
||||
fork(uid: string): AsyncGenerator {
|
||||
assert.strictEqual(this.#started, false, `can't create a fork after consuming the data`)
|
||||
assert.strictEqual(this.#forks.has(uid), false, `a fork with id ${uid} already exists`)
|
||||
|
||||
const fork = new Forked<T, TReturn, TNext>(this, uid)
|
||||
this.#forks.set(uid, fork)
|
||||
return fork
|
||||
@@ -85,7 +83,7 @@ export class Synchronized<T, TReturn, TNext> {
|
||||
// already removed
|
||||
return { done: true, value: undefined }
|
||||
}
|
||||
throw new Error(`trying to remove fork with uid ${uid} that is not a fork of this one`)
|
||||
throw new Error(`trying to remove fork wih uid ${uid} that is not a fork of this one`)
|
||||
}
|
||||
this.#forks.delete(uid)
|
||||
this.#waitingForks.delete(uid)
|
||||
@@ -144,14 +142,7 @@ class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
|
||||
return this.#parent.remove(this.#uid, e)
|
||||
}
|
||||
|
||||
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
|
||||
while (true) {
|
||||
const res = await this.next()
|
||||
if (res.done) {
|
||||
break
|
||||
}
|
||||
|
||||
yield res.value
|
||||
}
|
||||
[Symbol.asyncIterator](): AsyncGenerator<T> {
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,12 +139,4 @@ suite('error', () => {
|
||||
await first.next()
|
||||
assert.throws(() => forker.fork('second'))
|
||||
})
|
||||
test('It should not allow fork with the same uid', async () => {
|
||||
const progress = { yielded: 0 }
|
||||
const generator = makeRangeGenerator(5, progress)
|
||||
const forker = new Synchronized(generator)
|
||||
forker.fork('first')
|
||||
forker.fork('second')
|
||||
assert.throws(() => forker.fork('first'))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
|
||||
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
|
||||
|
||||
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ npm install --save @vates/json-hash
|
||||
|
||||
## Usage
|
||||
|
||||
The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
|
||||
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
|
||||
|
||||
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
|
||||
export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
|
||||
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negotiation
|
||||
export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
|
||||
export const NBD_OPT_EXPORT_NAME = 1
|
||||
export const NBD_OPT_ABORT = 2
|
||||
export const NBD_OPT_LIST = 3
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import assert from 'node:assert'
|
||||
import { Socket } from 'node:net'
|
||||
import { connect } from 'node:tls'
|
||||
import { pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
|
||||
import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
|
||||
import { readChunkStrict } from '@vates/read-chunk'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import {
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
OPTS_MAGIC,
|
||||
NBD_CMD_DISC,
|
||||
} from './constants.mjs'
|
||||
|
||||
const { warn } = createLogger('vates:nbd-client')
|
||||
|
||||
// documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
|
||||
@@ -43,9 +42,9 @@ export default class NbdClient {
|
||||
// AFAIK, there is no guaranty the server answers in the same order as the queries
|
||||
// so we handle a backlog of command waiting for response and handle concurrency manually
|
||||
|
||||
#waitingForResponse // there is already a listener waiting for a response
|
||||
#waitingForResponse // there is already a listenner waiting for a response
|
||||
#nextCommandQueryId = BigInt(0)
|
||||
#commandQueryBacklog // map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
|
||||
#commandQueryBacklog // map of command waiting for an response queryId => { size/*in byte*/, resolve, reject}
|
||||
#connected = false
|
||||
|
||||
#reconnectingPromise
|
||||
@@ -196,7 +195,7 @@ export default class NbdClient {
|
||||
}
|
||||
|
||||
// send export name we want to access.
|
||||
// it's implicitly closing the negotiation phase.
|
||||
// it's implictly closing the negociation phase.
|
||||
await this.#write(OPTS_MAGIC)
|
||||
await this.#writeInt32(NBD_OPT_EXPORT_NAME)
|
||||
const exportNameBuffer = Buffer.from(this.#exportName)
|
||||
@@ -219,22 +218,8 @@ export default class NbdClient {
|
||||
}
|
||||
|
||||
#write(buffer) {
|
||||
let timeout
|
||||
const messageTimeout = this.#messageTimeout
|
||||
const socket = this.#serverSocket
|
||||
return Promise.race([
|
||||
new Promise((resolve, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
reject(new Error('timeout'))
|
||||
}, messageTimeout)
|
||||
}),
|
||||
new Promise(resolve =>
|
||||
socket.write(buffer, () => {
|
||||
clearTimeout(timeout)
|
||||
resolve()
|
||||
})
|
||||
),
|
||||
])
|
||||
const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
|
||||
return pTimeout.call(promise, this.#messageTimeout)
|
||||
}
|
||||
|
||||
async #readInt32() {
|
||||
|
||||
@@ -30,8 +30,6 @@ export default class MultiNbdClient {
|
||||
* open nbdConcurrency connections to NBD servers
|
||||
* it must obtain at least one connection to succeed
|
||||
* it tries to spread connections on multiple host
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async connect() {
|
||||
const candidates = [...this.#settings]
|
||||
@@ -70,9 +68,7 @@ export default class MultiNbdClient {
|
||||
await _connect()
|
||||
}
|
||||
if (this.#clients.length === 0) {
|
||||
const error = new Error(`Fail to connect to any Nbd client`, { nbdInfos: this.#settings })
|
||||
error.code = 'NO_NBD_AVAILABLE'
|
||||
throw error
|
||||
throw new Error(`Fail to connect to any Nbd client`)
|
||||
}
|
||||
if (this.#clients.length < this.#nbdConcurrency) {
|
||||
warn(
|
||||
@@ -81,30 +77,17 @@ export default class MultiNbdClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async disconnect() {
|
||||
await asyncEach(this.#clients, client => client.disconnect(), {
|
||||
stopOnError: false,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} index
|
||||
* @param {number} size
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
|
||||
const clientId = index % this.#clients.length
|
||||
return this.#clients[clientId].readBlock(index, size)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {AsyncGenerator<Buffer>} indexGenerator
|
||||
*/
|
||||
async *readBlocks(indexGenerator) {
|
||||
// default : read all blocks
|
||||
const readAhead = []
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "3.1.4",
|
||||
"version": "3.1.2",
|
||||
"engines": {
|
||||
"node": ">=14.0"
|
||||
},
|
||||
@@ -24,7 +24,7 @@
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"xen-api": "^4.7.3"
|
||||
"xen-api": "^4.7.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tap": "^18.7.0",
|
||||
|
||||
@@ -24,7 +24,7 @@ $ npm install node-vsphere-soap --save
|
||||
|
||||
## Sample Code
|
||||
|
||||
### To connect to a vCenter server
|
||||
### To connect to a vCenter server:
|
||||
|
||||
var nvs = require('node-vsphere-soap');
|
||||
var vc = new nvs.Client(host, user, password, sslVerify);
|
||||
@@ -54,7 +54,7 @@ $ npm install node-vsphere-soap --save
|
||||
- userName - username of authenticated user
|
||||
- fullName - full name of authenticated user
|
||||
|
||||
### To run a command
|
||||
### To run a command:
|
||||
|
||||
var vcCmd = vc.runCommand( commandToRun, arguments );
|
||||
vcCmd.once('result', function( result, raw, soapHeader) {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
This file creates the Client class
|
||||
|
||||
- when the class is instantiated, a connection will be made to the ESXi/vCenter server to verify that the creds are good
|
||||
- upon a bad login, the connection will be terminated
|
||||
- upon a bad login, the connnection will be terminated
|
||||
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
### Usual workflow
|
||||
|
||||
> This section presents how this library should be used to implement a classic two factor authentication.
|
||||
> This section presents how this library should be used to implement a classic two factor authentification.
|
||||
|
||||
#### Setup
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ npm install --save @vates/otp
|
||||
|
||||
### Usual workflow
|
||||
|
||||
> This section presents how this library should be used to implement a classic two factor authentication.
|
||||
> This section presents how this library should be used to implement a classic two factor authentification.
|
||||
|
||||
#### Setup
|
||||
|
||||
|
||||
@@ -63,16 +63,16 @@ task.failure(error)
|
||||
Inside a task:
|
||||
|
||||
```js
|
||||
// the abort signal of the current task if any; otherwise, is `undefined`
|
||||
// the abort signal of the current task if any, otherwise is `undefined`
|
||||
Task.abortSignal
|
||||
|
||||
// sends an info on the current task if any; otherwise, does nothing
|
||||
// sends an info on the current task if any, otherwise does nothing
|
||||
Task.info(message, data)
|
||||
|
||||
// sends an info on the current task if any; otherwise, does nothing
|
||||
// sends an info on the current task if any, otherwise does nothing
|
||||
Task.warning(message, data)
|
||||
|
||||
// attaches a property to the current task if any; otherwise, does nothing
|
||||
// attaches a property to the current task if any, otherwise does nothing
|
||||
//
|
||||
// the latest value takes precedence
|
||||
//
|
||||
@@ -149,7 +149,7 @@ const onProgress = makeOnProgress({
|
||||
|
||||
// This function is called each time a root task or a subtask is updated.
|
||||
//
|
||||
// `taskLog.$root` can be used to unconditionally access the root task.
|
||||
// `taskLog.$root` can be used to uncondionally access the root task.
|
||||
onTaskUpdate(taskLog) {},
|
||||
})
|
||||
|
||||
|
||||
@@ -79,16 +79,16 @@ task.failure(error)
|
||||
Inside a task:
|
||||
|
||||
```js
|
||||
// the abort signal of the current task if any; otherwise, is `undefined`
|
||||
// the abort signal of the current task if any, otherwise is `undefined`
|
||||
Task.abortSignal
|
||||
|
||||
// sends an info on the current task if any; otherwise, does nothing
|
||||
// sends an info on the current task if any, otherwise does nothing
|
||||
Task.info(message, data)
|
||||
|
||||
// sends an info on the current task if any; otherwise, does nothing
|
||||
// sends an info on the current task if any, otherwise does nothing
|
||||
Task.warning(message, data)
|
||||
|
||||
// attaches a property to the current task if any; otherwise, does nothing
|
||||
// attaches a property to the current task if any, otherwise does nothing
|
||||
//
|
||||
// the latest value takes precedence
|
||||
//
|
||||
@@ -165,7 +165,7 @@ const onProgress = makeOnProgress({
|
||||
|
||||
// This function is called each time a root task or a subtask is updated.
|
||||
//
|
||||
// `taskLog.$root` can be used to unconditionally access the root task.
|
||||
// `taskLog.$root` can be used to uncondionally access the root task.
|
||||
onTaskUpdate(taskLog) {},
|
||||
})
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ function createTask(opts) {
|
||||
}
|
||||
|
||||
describe('Task', function () {
|
||||
describe('constructor', function () {
|
||||
describe('contructor', function () {
|
||||
it('data properties are passed to the start event', async function () {
|
||||
const properties = { foo: 0, bar: 1 }
|
||||
const task = createTask({ properties })
|
||||
@@ -265,7 +265,7 @@ describe('Task', function () {
|
||||
assert.throws(() => task.set(name, value), { message: /^task has not started yet\b/ })
|
||||
})
|
||||
|
||||
it(`emits a property message`, async function () {
|
||||
it(`emits an property message`, async function () {
|
||||
const task = createTask()
|
||||
await task.run(async () => {
|
||||
await Task.run(() => {
|
||||
@@ -339,7 +339,7 @@ describe('Task', function () {
|
||||
Task.warning('foo')
|
||||
})
|
||||
|
||||
it('emits a warning message when run inside a task', async function () {
|
||||
it('emits an warning message when run inside a task', async function () {
|
||||
const task = createTask()
|
||||
await task.run(() => {
|
||||
Task.warning('foo')
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.1",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
|
||||
@@ -5,8 +5,4 @@ import tseslint from 'typescript-eslint'
|
||||
|
||||
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
|
||||
ignores: ['dist'],
|
||||
rules: {
|
||||
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
|
||||
'@typescript-eslint/no-empty-object-type': 'off',
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@vates/types",
|
||||
"private": false,
|
||||
"version": "1.10.0",
|
||||
"version": "1.2.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": {
|
||||
@@ -17,10 +17,6 @@
|
||||
"./lib/vates/*": {
|
||||
"default": "./dist/lib/vates-*.mjs",
|
||||
"type": "./dist/lib/vates-*.d.mts"
|
||||
},
|
||||
"./lib/xen-orchestra/*": {
|
||||
"default": "./dist/lib/xen-orchestra-*.mjs",
|
||||
"type": "./dist/lib/xen-orchestra-*.d.mts"
|
||||
}
|
||||
},
|
||||
"type": "module",
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { XoHost } from './xo.mjs'
|
||||
|
||||
declare const __brand: unique symbol
|
||||
|
||||
export type Branded<TBrand extends string, TType = string> = TType & { [__brand]: TBrand }
|
||||
@@ -651,10 +649,6 @@ export const OPAQUE_REF = { EMPTY: 'OpaqueRef:NULL' } as const
|
||||
|
||||
export type OPAQUE_REF_NULL = (typeof OPAQUE_REF)['EMPTY']
|
||||
|
||||
export const BACKUP_TYPE = { backup: 'backup', metadata: 'metadataBackup', mirror: 'mirrorBackup' } as const
|
||||
|
||||
export type BACKUP_TYPE = (typeof BACKUP_TYPE)[keyof typeof BACKUP_TYPE]
|
||||
|
||||
// ----- XAPI Stats
|
||||
|
||||
type XapiStatsResponse<T> = {
|
||||
@@ -662,73 +656,47 @@ type XapiStatsResponse<T> = {
|
||||
interval: number
|
||||
stats: T
|
||||
}
|
||||
type StatValues = (number | null)[]
|
||||
type RecordStatValues = Record<string, StatValues>
|
||||
|
||||
export type XapiStatsGranularity = 'seconds' | 'minutes' | 'hours' | 'days'
|
||||
|
||||
export type XapiHostStatsRaw = {
|
||||
cpus?: RecordStatValues
|
||||
ioThroughput?: {
|
||||
r: RecordStatValues
|
||||
w: RecordStatValues
|
||||
export type XapiHostStats = XapiStatsResponse<{
|
||||
cpus: Record<string, number[]>
|
||||
ioThroughput: {
|
||||
r: Record<string, number[]>
|
||||
w: Record<string, number[]>
|
||||
}
|
||||
iops?: {
|
||||
r: RecordStatValues
|
||||
w: RecordStatValues
|
||||
iops: {
|
||||
r: Record<string, number[]>
|
||||
w: Record<string, number[]>
|
||||
}
|
||||
iowait?: RecordStatValues
|
||||
latency?: {
|
||||
r: RecordStatValues
|
||||
w: RecordStatValues
|
||||
iowait: Record<string, number[]>
|
||||
latency: {
|
||||
r: Record<string, number[]>
|
||||
w: Record<string, number[]>
|
||||
}
|
||||
load?: StatValues
|
||||
memory?: StatValues
|
||||
memoryFree?: StatValues
|
||||
pifs?: {
|
||||
rx: RecordStatValues
|
||||
tx: RecordStatValues
|
||||
load: number[]
|
||||
memory: number[]
|
||||
memoryFree: number[]
|
||||
pifs: {
|
||||
rx: Record<string, number[]>
|
||||
tx: Record<string, number[]>
|
||||
}
|
||||
}
|
||||
export type XapiHostStats = XapiStatsResponse<XapiHostStatsRaw>
|
||||
}>
|
||||
|
||||
export type XapiVmStatsRaw = {
|
||||
cpus?: RecordStatValues
|
||||
cpuUsage?: StatValues
|
||||
runstateFullrun?: StatValues
|
||||
runstateFullContention?: StatValues
|
||||
runstatePartialRun?: StatValues
|
||||
runstatePartialContention?: StatValues
|
||||
runstateConcurrencyHazard?: StatValues
|
||||
runstateBlocked?: StatValues
|
||||
iops?: {
|
||||
r: RecordStatValues
|
||||
w: RecordStatValues
|
||||
export type XapiVmStats = XapiStatsResponse<{
|
||||
cpus: Record<string, number[]>
|
||||
iops: {
|
||||
r: Record<string, number[]>
|
||||
w: Record<string, number[]>
|
||||
}
|
||||
memory?: StatValues
|
||||
memoryFree?: StatValues
|
||||
memoryTarget?: StatValues
|
||||
vifs?: {
|
||||
rx: RecordStatValues
|
||||
tx: RecordStatValues
|
||||
memory: number[]
|
||||
memoryFree?: number[]
|
||||
vifs: {
|
||||
rx: Record<string, number[]>
|
||||
tx: Record<string, number[]>
|
||||
}
|
||||
vifErrors?: {
|
||||
rx: RecordStatValues
|
||||
tx: RecordStatValues
|
||||
xvds: {
|
||||
w: Record<string, number[]>
|
||||
r: Record<string, number[]>
|
||||
}
|
||||
xvds?: {
|
||||
w?: RecordStatValues
|
||||
r?: RecordStatValues
|
||||
total?: RecordStatValues
|
||||
}
|
||||
vbdLatency?: {
|
||||
w: RecordStatValues
|
||||
r: RecordStatValues
|
||||
}
|
||||
vbdIowait?: RecordStatValues
|
||||
vbdInflight?: RecordStatValues
|
||||
vbdAvgquSz?: RecordStatValues
|
||||
}
|
||||
export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
|
||||
|
||||
export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>
|
||||
}>
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
export * from './common.mjs'
|
||||
export * from './xen-api.mjs'
|
||||
export * from './xo.mjs'
|
||||
export * from './lib/xen-orchestra-xapi.mjs'
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
import {
|
||||
WrappedXenApiRecord,
|
||||
XenApiHost,
|
||||
XenApiNetworkWrapped,
|
||||
XenApiRecord,
|
||||
XenApiSr,
|
||||
XenApiTask,
|
||||
XenApiVdi,
|
||||
XenApiVm,
|
||||
XenApiVmWrapped,
|
||||
} from '../xen-api.mjs'
|
||||
import type { Readable } from 'node:stream'
|
||||
import type {
|
||||
XoGpuGroup,
|
||||
XoVgpuType,
|
||||
XoHost,
|
||||
XoNetwork,
|
||||
XoPif,
|
||||
XoSr,
|
||||
XoUser,
|
||||
XoVdi,
|
||||
XoVm,
|
||||
XoVmTemplate,
|
||||
} from '../xo.mjs'
|
||||
|
||||
export type XcpPatches = {
|
||||
changelog?: {
|
||||
author: string
|
||||
date: number
|
||||
description: string
|
||||
}
|
||||
description: string
|
||||
license: string
|
||||
name: string
|
||||
release: string
|
||||
size: number
|
||||
url: string
|
||||
version: string
|
||||
}
|
||||
export type XsPatches = {
|
||||
conflicts?: string[]
|
||||
date: string
|
||||
description: string
|
||||
documentationUrl?: string
|
||||
guidances: string
|
||||
name: string
|
||||
id?: string
|
||||
paid?: boolean
|
||||
requirements?: string[]
|
||||
upgrade?: boolean
|
||||
url?: string
|
||||
uuid?: string
|
||||
}
|
||||
|
||||
export interface Xapi {
|
||||
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
|
||||
getField<T extends XenApiRecord, K extends keyof T>(
|
||||
type: Extract<WrappedXenApiRecord, T>['$type'],
|
||||
ref: T['$ref'],
|
||||
field: K
|
||||
): Promise<T[K]>
|
||||
createNetwork(
|
||||
params:
|
||||
| {
|
||||
name: string
|
||||
description?: string
|
||||
mtu?: number
|
||||
}
|
||||
| {
|
||||
name: string
|
||||
description?: string
|
||||
pifId: XoPif['id']
|
||||
mtu?: number
|
||||
/* between 0 and 4094 */
|
||||
vlan: number
|
||||
}
|
||||
): Promise<XenApiNetworkWrapped>
|
||||
deleteNetwork(id: XoNetwork['id']): Promise<void>
|
||||
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
|
||||
pool_emergencyShutdown(): Promise<void>
|
||||
resumeVm(id: XoVm['id']): Promise<void>
|
||||
unpauseVm(id: XoVm['id']): Promise<void>
|
||||
startVm(
|
||||
id: XoVm['id'],
|
||||
opts?: {
|
||||
bypassMacAddressesCheck?: boolean
|
||||
force?: boolean
|
||||
hostId?: XoHost['id']
|
||||
/**
|
||||
* if startOnly is true and the VM is not halted, throw VM_BAD_POWER_STATE
|
||||
* otherwise, unpause/resume the VM
|
||||
*
|
||||
* @default false
|
||||
*/
|
||||
startOnly?: boolean
|
||||
}
|
||||
): Promise<void>
|
||||
VM_import(
|
||||
stream: Readable,
|
||||
srRef?: XenApiSr['$ref'],
|
||||
onVmCreation?: null | ((vm: XenApiVm) => unknown)
|
||||
): Promise<XenApiVm['$ref']>
|
||||
createVm(
|
||||
templateUuid: XoVmTemplate['uuid'],
|
||||
metadataVm: {
|
||||
affinityHost?: XoHost['id']
|
||||
autoPoweron?: boolean
|
||||
memory?: number
|
||||
name_description?: string
|
||||
name_label: string
|
||||
nameLabel?: string
|
||||
clone?: boolean
|
||||
installRepository?: XoVdi['id'] | '' | null
|
||||
vdis?: (
|
||||
| /** Create VDI */ {
|
||||
name_label: string
|
||||
size: number
|
||||
sr?: XoSr['id']
|
||||
name_description?: string
|
||||
}
|
||||
| /** Update VDI*/ {
|
||||
userdevice: string
|
||||
name_label?: string
|
||||
size?: number
|
||||
sr?: XoSr['id']
|
||||
name_description?: string
|
||||
}
|
||||
| /** Destroy VDI*/ { destroy: true; userdevice: string }
|
||||
)[]
|
||||
vifs?: (
|
||||
| /** Create/update VIF */ {
|
||||
device?: string
|
||||
ipv4_allowed?: string[]
|
||||
ipv6_allowed?: string[]
|
||||
mac?: string
|
||||
mtu?: number
|
||||
network: string
|
||||
}
|
||||
| /** Destroy VIF */ { destroy: true; device: string }
|
||||
)[]
|
||||
existingVdis?: {
|
||||
$SR: XoSr['id']
|
||||
size: number
|
||||
userdevice: string
|
||||
}[]
|
||||
vgpuType?: XoVgpuType['id']
|
||||
gpuGroup?: XoGpuGroup['id']
|
||||
copyHostBiosStrings?: boolean
|
||||
},
|
||||
checkLimits?: boolean,
|
||||
creatorId?: XoUser['id'],
|
||||
opts?: { destroyAllVifs: boolean }
|
||||
): Promise<XenApiVmWrapped>
|
||||
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
|
||||
VM_createCloudInitConfig(
|
||||
vmRef: XenApiVm['$ref'],
|
||||
cloudConfig: string,
|
||||
opts?: { networkConfig?: string }
|
||||
): Promise<XoVdi['uuid']>
|
||||
VM_destroy(
|
||||
vmRef: XenApiVm['$ref'],
|
||||
opts?: {
|
||||
deleteDisks?: boolean
|
||||
force?: boolean
|
||||
bypassBlockedOperation?: boolean
|
||||
forceDeleteDefaultTemplate?: boolean
|
||||
}
|
||||
): Promise<void>
|
||||
getResource(
|
||||
pathname: string,
|
||||
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
|
||||
): Promise<{ body: Readable }>
|
||||
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
|
||||
}
|
||||
@@ -64,7 +64,6 @@ import type {
|
||||
VTPM_OPERATIONS,
|
||||
VUSB_OPERATIONS,
|
||||
} from './common.mjs'
|
||||
import type { Xapi } from './lib/xen-orchestra-xapi.mjs'
|
||||
|
||||
// types automatically generated by the @ByScripts script.
|
||||
// https://github.com/vatesfr/xen-orchestra/tree/wip-xapi-types-generator/%40xen-orchestra/xapi-generator
|
||||
@@ -78,13 +77,10 @@ type WrapperXenApi<T, Type extends string, Fn = { (): void }> = T & {
|
||||
$call: Fn
|
||||
$callAsync: Fn
|
||||
$type: Type
|
||||
$snapshot(params: {
|
||||
cancelToken?: unknown
|
||||
ignoredVdisTag?: string
|
||||
name_label?: string
|
||||
unplugVusbs?: boolean
|
||||
}): Promise<XenApiVm['$ref']>
|
||||
$xapi: Xapi
|
||||
$xapi: {
|
||||
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
}
|
||||
}
|
||||
|
||||
export interface XenApiSession {
|
||||
@@ -258,14 +254,6 @@ export interface XenApiPoolUpdate {
|
||||
|
||||
type XenApiVmCallMethods = {
|
||||
(method: 'start', start_paused: boolean, force: boolean): Promise<void>
|
||||
(method: 'clean_shutdown'): Promise<void>
|
||||
(method: 'hard_shutdown'): Promise<void>
|
||||
(method: 'clean_reboot'): Promise<void>
|
||||
(method: 'hard_reboot'): Promise<void>
|
||||
(method: 'pause'): Promise<void>
|
||||
(method: 'suspend'): Promise<void>
|
||||
(method: 'record_data_source', dataSource: string): Promise<void>
|
||||
(method: 'forget_data_source_archives', dataSource: string): Promise<void>
|
||||
}
|
||||
export interface XenApiVm {
|
||||
$ref: Branded<'VM'>
|
||||
@@ -633,7 +621,6 @@ export interface XenApiNetwork {
|
||||
uuid: string
|
||||
VIFs: XenApiVif['$ref'][]
|
||||
}
|
||||
export interface XenApiNetworkWrapped extends WrapperXenApi<XenApiNetwork, 'network'> {}
|
||||
|
||||
export interface XenApiVif {
|
||||
$ref: Branded<'VIF'>
|
||||
@@ -718,7 +705,6 @@ export interface XenApiPif {
|
||||
VLAN_slave_of: XenApiVlan['$ref'][]
|
||||
VLAN: number
|
||||
}
|
||||
export type XenApiPifWrapped = WrapperXenApi<XenApiPif, 'PIF'>
|
||||
|
||||
export interface XenApiPifMetrics {
|
||||
$ref: Branded<'PIF_metrics'>
|
||||
@@ -774,13 +760,11 @@ export interface XenApiSm {
|
||||
other_config: Record<string, string>
|
||||
required_api_version: string
|
||||
required_cluster_stack: string[]
|
||||
supported_image_formats?: string[]
|
||||
type: string
|
||||
uuid: string
|
||||
vendor: string
|
||||
version: string
|
||||
}
|
||||
export type XenApiSmWrapped = WrapperXenApi<XenApiSm, 'SM'>
|
||||
|
||||
export interface XenApiSr {
|
||||
$ref: Branded<'SR'>
|
||||
@@ -893,7 +877,7 @@ export interface XenApiVbd {
|
||||
unpluggable: boolean
|
||||
userdevice: string
|
||||
uuid: string
|
||||
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
|
||||
VDI: XenApiVdi['$ref']
|
||||
VM: XenApiVm['$ref']
|
||||
}
|
||||
export type XenApiVbdWrapped = WrapperXenApi<XenApiVbd, 'VBD'>
|
||||
@@ -1036,7 +1020,6 @@ export interface XenApiPci {
|
||||
uuid: string
|
||||
vendor_name: string
|
||||
}
|
||||
export type XenApiPciWrapped = WrapperXenApi<XenApiPci, 'PCI'>
|
||||
|
||||
export interface XenApiPgpu {
|
||||
$ref: Branded<'PGPU'>
|
||||
@@ -1053,7 +1036,6 @@ export interface XenApiPgpu {
|
||||
supported_VGPU_types: XenApiVgpuType['$ref'][]
|
||||
uuid: string
|
||||
}
|
||||
export type XenApiPgpuWrapped = WrapperXenApi<XenApiPgpu, 'PGPU'>
|
||||
|
||||
export interface XenApiGpuGroup {
|
||||
$ref: Branded<'GPU_group'>
|
||||
@@ -1068,7 +1050,6 @@ export interface XenApiGpuGroup {
|
||||
uuid: string
|
||||
VGPUs: XenApiVgpu['$ref'][]
|
||||
}
|
||||
export type XenApiGpuGroupWrapped = WrapperXenApi<XenApiGpuGroup, 'gpuGroup'>
|
||||
|
||||
export interface XenApiVgpu {
|
||||
$ref: Branded<'VGPU'>
|
||||
@@ -1106,7 +1087,6 @@ export interface XenApiVgpuType {
|
||||
vendor_name: string
|
||||
VGPUs: XenApiVgpu['$ref'][]
|
||||
}
|
||||
export type XenApiVgpuTypeWrapped = WrapperXenApi<XenApiVgpuType, 'vgpuType'>
|
||||
|
||||
export interface XenApiPvsSite {
|
||||
$ref: Branded<'PVS_site'>
|
||||
@@ -1356,8 +1336,6 @@ export type XenApiRecord =
|
||||
|
||||
export type WrappedXenApiRecord =
|
||||
| XenApiHostWrapped
|
||||
| XenApiNetworkWrapped
|
||||
| XenApiPifWrapped
|
||||
| XenApiPoolWrapped
|
||||
| XenApiSrWrapped
|
||||
| XenApiVbdWrapped
|
||||
@@ -1366,7 +1344,3 @@ export type WrappedXenApiRecord =
|
||||
| XenApiVifWrapped
|
||||
| XenApiVmWrapped
|
||||
| XenApiVtpmWrapped
|
||||
| XenApiPciWrapped
|
||||
| XenApiGpuGroupWrapped
|
||||
| XenApiPgpuWrapped
|
||||
| XenApiVgpuTypeWrapped
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
// Types based on xapi-object-to-xo
|
||||
|
||||
import type {
|
||||
BACKUP_TYPE,
|
||||
Branded,
|
||||
DOMAIN_TYPE,
|
||||
HOST_ALLOWED_OPERATIONS,
|
||||
HOST_POWER_STATE,
|
||||
IP_CONFIGURATION_MODE,
|
||||
IPV6_CONFIGURATION_MODE,
|
||||
NETWORK_OPERATIONS,
|
||||
PGPU_DOM0_ACCESS,
|
||||
POOL_ALLOWED_OPERATIONS,
|
||||
PRIMARY_ADDRESS_TYPE,
|
||||
STORAGE_OPERATIONS,
|
||||
VDI_OPERATIONS,
|
||||
VDI_TYPE,
|
||||
@@ -75,7 +70,7 @@ type BaseXoVm = BaseXapiXo & {
|
||||
isNestedVirtEnabled: boolean
|
||||
hasVendorDevice: boolean
|
||||
high_availability: string
|
||||
installTime?: number | null
|
||||
installTime?: number
|
||||
isFirmwareSupported: boolean
|
||||
memory: {
|
||||
dynamic: number[]
|
||||
@@ -101,7 +96,7 @@ type BaseXoVm = BaseXapiXo & {
|
||||
secureBoot: boolean
|
||||
snapshots: XoVmSnapshot['id'][]
|
||||
startDelay: number
|
||||
startTime?: number | null
|
||||
startTime?: number
|
||||
suspendSr?: XoSr['id']
|
||||
tags: string[]
|
||||
vga?: string
|
||||
@@ -123,7 +118,7 @@ type BaseXoVm = BaseXapiXo & {
|
||||
|
||||
export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
|
||||
body: {
|
||||
value?: string
|
||||
value: string
|
||||
name: string
|
||||
}
|
||||
object: {
|
||||
@@ -133,22 +128,6 @@ export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
|
||||
}
|
||||
}
|
||||
|
||||
export type XoBackupRepository = {
|
||||
benchmarks?: { readRate: number; timestamp: number; writeRate: number }[]
|
||||
enabled: boolean
|
||||
error?: Record<string, unknown>
|
||||
id: Branded<'backup-repository'>
|
||||
name: string
|
||||
options?: string
|
||||
proxy?: XoProxy['id']
|
||||
url: string
|
||||
}
|
||||
|
||||
export type XoGpuGroup = BaseXapiXo & {
|
||||
id: Branded<'gpu-group'>
|
||||
type: 'gpuGroup'
|
||||
}
|
||||
|
||||
export type XoGroup = {
|
||||
id: Branded<'group'>
|
||||
name: string
|
||||
@@ -282,65 +261,18 @@ export type XoPbd = BaseXapiXo & {
|
||||
}
|
||||
|
||||
export type XoPci = BaseXapiXo & {
|
||||
$host?: XoHost['id']
|
||||
|
||||
class_name: string
|
||||
device_name: string
|
||||
id: Branded<'PCI'>
|
||||
pci_id: string
|
||||
type: 'PCI'
|
||||
}
|
||||
|
||||
export type XoPgpu = BaseXapiXo & {
|
||||
$host: XoHost['id']
|
||||
$vgpus: XoVgpu['id'][]
|
||||
|
||||
dom0Access: PGPU_DOM0_ACCESS
|
||||
enabledVgpuTypes: XoVgpuType['id'][]
|
||||
gpuGroup?: XoGpuGroup['id']
|
||||
host: XoHost['id']
|
||||
id: Branded<'PGPU'>
|
||||
isSystemDisplayDevice: boolean
|
||||
pci?: XoPci['id']
|
||||
// it seems that supportedVgpuMaxCapcities always return undefined (See: xapi-objet-to-xo)
|
||||
/**
|
||||
* @deprecated
|
||||
*/
|
||||
supportedVgpuMaxCapcities?: never
|
||||
supportedVgpuTypes: XoVgpuType['id'][]
|
||||
type: 'PGPU'
|
||||
vgpus: XoVgpu['id'][]
|
||||
}
|
||||
|
||||
export type XoPif = BaseXapiXo & {
|
||||
$host: XoHost['id']
|
||||
$network: XoNetwork['id']
|
||||
|
||||
attached: boolean
|
||||
bondMaster?: XoPif['id']
|
||||
bondSalves?: XoPif['id'][]
|
||||
carrier: boolean
|
||||
device: string
|
||||
deviceName?: string
|
||||
disallowUnplug: boolean
|
||||
dns: string
|
||||
gateway: string
|
||||
id: Branded<'PIF'>
|
||||
ip: string
|
||||
ipv6: string[]
|
||||
ipv6Mode: IPV6_CONFIGURATION_MODE
|
||||
isBondMaster: boolean
|
||||
isBondSlave: boolean
|
||||
mac: string
|
||||
management: boolean
|
||||
mode: IP_CONFIGURATION_MODE
|
||||
mtu: number
|
||||
netmask: string
|
||||
physical: boolean
|
||||
primaryAddressType: PRIMARY_ADDRESS_TYPE
|
||||
speed?: number
|
||||
type: 'PIF'
|
||||
vlan: number
|
||||
}
|
||||
|
||||
export type XoPool = BaseXapiXo & {
|
||||
@@ -369,65 +301,15 @@ export type XoPool = BaseXapiXo & {
|
||||
zstdSupported: boolean
|
||||
}
|
||||
|
||||
export type XoProxy = {
|
||||
id: Branded<'proxy'>
|
||||
}
|
||||
|
||||
type BaseXoJob = {
|
||||
export type XoJob = {
|
||||
id: Branded<'job'>
|
||||
}
|
||||
// @TODO: create type for complex matcher
|
||||
export type XoBackupJob = BaseXoJob & {
|
||||
compression?: 'native' | 'zstd' | ''
|
||||
proxy?: XoProxy['id']
|
||||
mode: 'full' | 'delta'
|
||||
name?: string
|
||||
remotes?: {
|
||||
id: XoBackupRepository['id'] | { __or: XoBackupRepository['id'][] }
|
||||
}
|
||||
vms?: {
|
||||
id: XoVm['id'] | { __or: XoVm['id'][] } | Record<string, unknown>
|
||||
}
|
||||
srs: {
|
||||
id: XoSr['id'] | { __or: XoSr['id'][] }
|
||||
}
|
||||
type: BACKUP_TYPE
|
||||
settings: {
|
||||
'': {
|
||||
cbtDestroySnapshotData?: boolean
|
||||
concurrency?: number
|
||||
longTermRetention?: {
|
||||
daily?: { retention: number; settings: Record<string, unknown> }
|
||||
weekly?: { retention: number; settings: Record<string, unknown> }
|
||||
monthly?: { retention: number; settings: Record<string, unknown> }
|
||||
yearly?: { retention: number; settings: Record<string, unknown> }
|
||||
}
|
||||
maxExportRate?: number
|
||||
nbdConcurrency?: number
|
||||
nRetriesVmBackupFailures?: number
|
||||
preferNbd?: boolean
|
||||
timezone?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
[key: XoSchedule['id']]: {
|
||||
exportRetention?: number
|
||||
healthCheckSr?: XoSr['id']
|
||||
healthCheckVmsWithTags?: string[]
|
||||
fullInterval?: number
|
||||
copyRetention?: number
|
||||
snapshotRetention?: number
|
||||
cbtDestroySnapshotData?: boolean
|
||||
[key: string]: unknown
|
||||
}
|
||||
}
|
||||
}
|
||||
export type XoJob = BaseXoJob & {}
|
||||
|
||||
export type XoSchedule = {
|
||||
cron: string
|
||||
enabled: boolean
|
||||
enable: boolean
|
||||
id: Branded<'schedule'>
|
||||
jobId: (XoJob | XoBackupJob)['id']
|
||||
jobId: XoJob['id']
|
||||
name?: string
|
||||
timezone?: string
|
||||
}
|
||||
@@ -440,7 +322,6 @@ export type XoServer = {
|
||||
httpProxy?: string
|
||||
id: Branded<'server'>
|
||||
label?: string
|
||||
master?: XoHost['id']
|
||||
poolId?: XoPool['id']
|
||||
poolNameDescription?: string
|
||||
poolNameLabel?: string
|
||||
@@ -454,7 +335,7 @@ export type XoSr = BaseXapiXo & {
|
||||
|
||||
$container: XoPool['id'] | XoHost['id']
|
||||
|
||||
VDIs: AnyXoVdi['id'][]
|
||||
VDIs: XoVdi['id'][]
|
||||
|
||||
allocationStrategy: 'thin' | 'thick' | 'unknown'
|
||||
content_type: string
|
||||
@@ -464,9 +345,9 @@ export type XoSr = BaseXapiXo & {
|
||||
name_description: string
|
||||
name_label: string
|
||||
other_config: Record<string, string>
|
||||
physical_usage: number
|
||||
physical_usage: number | null
|
||||
shared: boolean
|
||||
size: number
|
||||
size: number | null
|
||||
sm_config: Record<string, string>
|
||||
SR_type: string
|
||||
tags: string[]
|
||||
@@ -474,21 +355,6 @@ export type XoSr = BaseXapiXo & {
|
||||
usage: number
|
||||
}
|
||||
|
||||
export type XoSm = BaseXapiXo & {
|
||||
id: Branded<'SM'>
|
||||
|
||||
SM_type: string
|
||||
vendor: string
|
||||
name_label: string
|
||||
name_description: string
|
||||
configuration: Record<string, string>
|
||||
features: Record<string, number>
|
||||
driver_filename: string
|
||||
required_cluster_stack: string[]
|
||||
supported_image_formats: string[]
|
||||
type: 'SM'
|
||||
}
|
||||
|
||||
export type XoUser = {
|
||||
authProviders?: Record<string, string>
|
||||
email: string
|
||||
@@ -509,8 +375,8 @@ export type XoVbd = BaseXapiXo & {
|
||||
position: string
|
||||
read_only: boolean
|
||||
type: 'VBD'
|
||||
VDI?: XoVdi['id'] | XoVdiSnapshot['id']
|
||||
VM: AnyXoVm['id']
|
||||
VDI: XoVdi['id']
|
||||
VM: XoVm['id']
|
||||
}
|
||||
|
||||
type BaseXoVdi = BaseXapiXo & {
|
||||
@@ -526,7 +392,6 @@ type BaseXoVdi = BaseXapiXo & {
|
||||
name_label: string
|
||||
other_config: Record<string, string>
|
||||
parent?: XoVdiUnmanaged['id']
|
||||
image_format?: string
|
||||
size: number
|
||||
snapshots: XoVdiSnapshot['id'][]
|
||||
tags: string[]
|
||||
@@ -545,24 +410,14 @@ export type XoVdiSnapshot = BaseXoVdi & {
|
||||
type: 'VDI-snapshot'
|
||||
}
|
||||
|
||||
export type XoVdiUnmanaged = Omit<BaseXoVdi, '$VBDs'> & {
|
||||
export type XoVdiUnmanaged = BaseXoVdi & {
|
||||
id: Branded<'VDI-unmanaged'>
|
||||
type: 'VDI-unmanaged'
|
||||
|
||||
/**
|
||||
* Unmanaged VDI have no VBDs
|
||||
*/
|
||||
$VBDs: never[]
|
||||
}
|
||||
|
||||
export type XoVgpu = BaseXapiXo & {
|
||||
id: Branded<'VGPU'>
|
||||
type: 'vgpu'
|
||||
}
|
||||
|
||||
export type XoVgpuType = BaseXapiXo & {
|
||||
id: Branded<'vgpu-type'>
|
||||
type: 'vgpuType'
|
||||
type: 'VGPU'
|
||||
}
|
||||
|
||||
export type XoVif = BaseXapiXo & {
|
||||
@@ -590,7 +445,6 @@ export type XoVif = BaseXapiXo & {
|
||||
export type XoVm = BaseXoVm & {
|
||||
id: Branded<'VM'>
|
||||
type: 'VM'
|
||||
vulnerabilities: { xsa468: boolean | { reason: string; driver?: string; version?: string } }
|
||||
}
|
||||
|
||||
export type XoVmController = BaseXoVm & {
|
||||
@@ -631,13 +485,9 @@ export type XoVtpm = BaseXapiXo & {
|
||||
|
||||
export type XapiXoRecord =
|
||||
| XoAlarm
|
||||
| XoGpuGroup
|
||||
| XoHost
|
||||
| XoMessage
|
||||
| XoNetwork
|
||||
| XoPci
|
||||
| XoPgpu
|
||||
| XoPif
|
||||
| XoPool
|
||||
| XoSr
|
||||
| XoVbd
|
||||
@@ -645,21 +495,13 @@ export type XapiXoRecord =
|
||||
| XoVdiSnapshot
|
||||
| XoVdiUnmanaged
|
||||
| XoVgpu
|
||||
| XoVgpuType
|
||||
| XoVif
|
||||
| XoVm
|
||||
| XoVmController
|
||||
| XoVmSnapshot
|
||||
| XoVmTemplate
|
||||
| XoVtpm
|
||||
| XoSm
|
||||
|
||||
export type NonXapiXoRecord = XoGroup | XoProxy | XoJob | XoBackupRepository | XoSchedule | XoServer | XoUser
|
||||
export type NonXapiXoRecord = XoGroup | XoJob | XoSchedule | XoServer | XoUser
|
||||
|
||||
export type XoRecord = XapiXoRecord | NonXapiXoRecord
|
||||
|
||||
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
|
||||
|
||||
export type AnyXoVdi = XoVdi | XoVdiSnapshot | XoVdiUnmanaged
|
||||
|
||||
export type AnyXoJob = XoJob | XoBackupJob
|
||||
|
||||
@@ -20,7 +20,7 @@ function main([inputPath = 0]) {
|
||||
try {
|
||||
data = JSON.parse(input)
|
||||
} catch (error) {
|
||||
// fall back to XML
|
||||
// fallback to XML
|
||||
log(parseXml(input))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,10 +4,6 @@ Usage:
|
||||
|
||||
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
|
||||
|
||||
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
|
||||
|
||||
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
|
||||
|
||||
Detects and repair issues with VM backups.
|
||||
|
||||
Options:
|
||||
|
||||
@@ -20,10 +20,6 @@ Usage:
|
||||
|
||||
xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
|
||||
|
||||
alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
|
||||
|
||||
xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
|
||||
|
||||
Detects and repair issues with VM backups.
|
||||
|
||||
Options:
|
||||
|
||||
@@ -1,81 +1,13 @@
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import getopts from 'getopts'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { basename, dirname } from 'path'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { pathToFileURL } from 'url'
|
||||
|
||||
async function cleanFiles(vmDirs, { fix, merge, remove }) {
|
||||
let nbSucceed = 0
|
||||
let nbFailed = 0
|
||||
let nbStarted = 0
|
||||
const total = vmDirs.length
|
||||
// don't assume all the files uses the same remote
|
||||
await asyncEach(
|
||||
vmDirs,
|
||||
vmDir =>
|
||||
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
|
||||
try {
|
||||
nbStarted++
|
||||
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
|
||||
fixMetadata: fix,
|
||||
remove,
|
||||
merge,
|
||||
logInfo: (...args) => console.log(...args),
|
||||
logWarn: (...args) => console.warn(...args),
|
||||
})
|
||||
nbSucceed++
|
||||
} catch (error) {
|
||||
nbFailed++
|
||||
console.error('adapter.cleanVm', vmDir, error, { nbStarted, nbSucceed, nbFailed, total })
|
||||
}
|
||||
}),
|
||||
{ concurrency: 2 }
|
||||
)
|
||||
console.log('done', { nbSucceed, nbFailed })
|
||||
}
|
||||
|
||||
async function cleanRemote(url, { fix, merge, remove }) {
|
||||
let nbSucceed = 0
|
||||
let nbFailed = 0
|
||||
let nbStarted = 0
|
||||
await Disposable.use(getSyncedHandler({ url }), async handler => {
|
||||
const remoteAdapter = new RemoteAdapter(handler)
|
||||
const vmUuids = await remoteAdapter.listAllVms()
|
||||
const total = vmUuids.length
|
||||
await asyncEach(
|
||||
vmUuids,
|
||||
async vmUuid => {
|
||||
try {
|
||||
nbStarted++
|
||||
await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
|
||||
fixMetadata: fix,
|
||||
remove,
|
||||
merge,
|
||||
logInfo: (...args) => console.log(...args),
|
||||
logWarn: (...args) => console.warn(...args),
|
||||
})
|
||||
nbSucceed++
|
||||
} catch (error) {
|
||||
nbFailed++
|
||||
console.error('adapter.cleanVm', vmUuid, error, { nbStarted, nbSucceed, nbFailed, total })
|
||||
}
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
)
|
||||
})
|
||||
console.log('done', { nbSucceed, nbFailed })
|
||||
}
|
||||
|
||||
export default async function cleanVms(args) {
|
||||
const {
|
||||
_: vmDirs,
|
||||
fix,
|
||||
remote,
|
||||
remove,
|
||||
merge,
|
||||
} = getopts(args, {
|
||||
const { _, fix, remove, merge } = getopts(args, {
|
||||
alias: {
|
||||
fix: 'f',
|
||||
remove: 'r',
|
||||
@@ -88,9 +20,19 @@ export default async function cleanVms(args) {
|
||||
},
|
||||
})
|
||||
|
||||
if (remote) {
|
||||
return cleanRemote(remote, { fix, merge, remove })
|
||||
} else {
|
||||
return cleanFiles(vmDirs, { fix, merge, remove })
|
||||
}
|
||||
await asyncMap(_, vmDir =>
|
||||
Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
|
||||
try {
|
||||
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
|
||||
fixMetadata: fix,
|
||||
remove,
|
||||
merge,
|
||||
logInfo: (...args) => console.log(...args),
|
||||
logWarn: (...args) => console.warn(...args),
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('adapter.cleanVm', vmDir, error)
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6,10 +6,9 @@
|
||||
"preferGlobal": true,
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/backups": "^0.63.0",
|
||||
"@xen-orchestra/fs": "^4.6.2",
|
||||
"@xen-orchestra/backups": "^0.59.0",
|
||||
"@xen-orchestra/fs": "^4.5.0",
|
||||
"filenamify": "^6.0.0",
|
||||
"getopts": "^2.2.5",
|
||||
"lodash": "^4.17.15",
|
||||
@@ -28,7 +27,7 @@
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"version": "1.1.2",
|
||||
"version": "1.0.30",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
|
||||
@@ -4,13 +4,12 @@ import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { importIncrementalVm } from './_incrementalVm.mjs'
|
||||
import { Task } from './Task.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
import { VhdNegative, VhdSynthetic } from 'vhd-lib'
|
||||
import { decorateClass } from '@vates/decorate-with'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { dirname, join } from 'node:path'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
import { defer } from 'golike-defer'
|
||||
import { NegativeDisk } from '@xen-orchestra/disk-transform'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
|
||||
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
|
||||
async function resolveUuid(xapi, cache, uuid, type) {
|
||||
@@ -60,7 +59,7 @@ export class ImportVmBackup {
|
||||
const metadata = this._metadata
|
||||
const { mapVdisSrs } = this._importIncrementalVmSettings
|
||||
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
|
||||
const disks = {}
|
||||
const streams = {}
|
||||
const metdataDir = dirname(metadata._filename)
|
||||
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
|
||||
|
||||
@@ -111,21 +110,20 @@ export class ImportVmBackup {
|
||||
}
|
||||
}
|
||||
|
||||
let disk
|
||||
let stream
|
||||
const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
|
||||
if (vhdPath === backupWithSnapshotPath) {
|
||||
// all the data are already on the host
|
||||
debug('direct reuse of a snapshot')
|
||||
disk = null
|
||||
stream = null
|
||||
vdis[vdiRef].baseVdi = snapshotCandidate
|
||||
// go next disk , we won't use this stream
|
||||
continue
|
||||
}
|
||||
|
||||
const parent = await openDiskChain({
|
||||
handler: this._adapter._handler,
|
||||
path: vhdPath,
|
||||
})
|
||||
let disposableDescendants
|
||||
|
||||
const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
|
||||
|
||||
// this will also clean if another disk of this VM backup fails
|
||||
// if user really only need to restore non failing disks he can retry with ignoredVdis
|
||||
@@ -134,7 +132,8 @@ export class ImportVmBackup {
|
||||
if (!disposed) {
|
||||
disposed = true
|
||||
try {
|
||||
await parent?.close()
|
||||
await disposableDescendants?.dispose()
|
||||
await disposableSynthetic?.dispose()
|
||||
} catch (error) {
|
||||
warn('openVhd: failed to dispose VHDs', { error })
|
||||
}
|
||||
@@ -142,10 +141,11 @@ export class ImportVmBackup {
|
||||
}
|
||||
$defer.onFailure(() => disposeOnce())
|
||||
|
||||
debug('got vhd synthetic of parents', parent)
|
||||
const parentVhd = disposableSynthetic.value
|
||||
await parentVhd.readBlockAllocationTable()
|
||||
debug('got vhd synthetic of parents', parentVhd.length)
|
||||
|
||||
if (snapshotCandidate !== undefined) {
|
||||
let descendant, negativeDisk
|
||||
try {
|
||||
debug('will try to use differential restore', {
|
||||
backupWithSnapshotPath,
|
||||
@@ -153,37 +153,39 @@ export class ImportVmBackup {
|
||||
vdiRef,
|
||||
})
|
||||
|
||||
descendant = await openDiskChain({
|
||||
handler: this._adapter._handler,
|
||||
path: backupWithSnapshotPath,
|
||||
disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
|
||||
until: vhdPath,
|
||||
})
|
||||
|
||||
const descendantsVhd = disposableDescendants.value
|
||||
await descendantsVhd.readBlockAllocationTable()
|
||||
debug('got vhd synthetic of descendants')
|
||||
negativeDisk = new NegativeDisk(parent, descendant)
|
||||
const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
|
||||
debug('got vhd negative')
|
||||
|
||||
// update the stream with the negative vhd stream
|
||||
disk = negativeDisk
|
||||
stream = await negativeVhd.stream()
|
||||
vdis[vdiRef].baseVdi = snapshotCandidate
|
||||
} catch (error) {
|
||||
// can be a broken VHD chain, a vhd chain with a key backup, ....
|
||||
// not an irrecuperable error, don't dispose parentVhd, and fall back to full restore
|
||||
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
|
||||
warn(`can't use differential restore`, { error })
|
||||
descendant?.close()
|
||||
negativeDisk?.close()
|
||||
disposableDescendants?.dispose()
|
||||
}
|
||||
}
|
||||
// didn't make a negative stream : fall back to classic stream
|
||||
if (disk === undefined) {
|
||||
// didn't make a negative stream : fallback to classic stream
|
||||
if (stream === undefined) {
|
||||
debug('use legacy restore')
|
||||
disk = parent
|
||||
stream = await parentVhd.stream()
|
||||
}
|
||||
info('everything is ready, will transfer', disk)
|
||||
disks[vdiRef] = disk
|
||||
|
||||
stream.on('end', disposeOnce)
|
||||
stream.on('close', disposeOnce)
|
||||
stream.on('error', disposeOnce)
|
||||
info('everything is ready, will transfer', stream.length)
|
||||
streams[`${vdiRef}.vhd`] = stream
|
||||
}
|
||||
return {
|
||||
disks,
|
||||
streams,
|
||||
vbds,
|
||||
vdis,
|
||||
version: '1.0.0',
|
||||
@@ -239,6 +241,7 @@ export class ImportVmBackup {
|
||||
assert.strictEqual(metadata.mode, 'delta')
|
||||
|
||||
backup = await this.#decorateIncrementalVmMetadata()
|
||||
Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
|
||||
}
|
||||
|
||||
return Task.run(
|
||||
|
||||
@@ -2,7 +2,7 @@ import { asyncEach } from '@vates/async-each'
|
||||
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import { compose } from '@vates/compose'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
|
||||
import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
|
||||
import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { deduped } from '@vates/disposable/deduped.js'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
@@ -10,6 +10,7 @@ import { execFile } from 'child_process'
|
||||
import { mount } from '@vates/fuse-vhd'
|
||||
import { readdir, lstat } from 'node:fs/promises'
|
||||
import { synchronized } from 'decorator-synchronized'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { ZipFile } from 'yazl'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import fromCallback from 'promise-toolbox/fromCallback'
|
||||
@@ -31,11 +32,6 @@ import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
|
||||
import { lvs, pvs } from './_lvm.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
|
||||
import { RemoteVhd } from './disks/RemoteVhd.mjs'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { ReadAhead } from '@xen-orchestra/disk-transform'
|
||||
|
||||
export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
|
||||
|
||||
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
|
||||
@@ -205,7 +201,7 @@ export class RemoteAdapter {
|
||||
})
|
||||
}
|
||||
|
||||
// check if we will be allowed to merge a vhd created in this adapter
|
||||
// check if we will be allowed to merge a a vhd created in this adapter
|
||||
// with the vhd at path `path`
|
||||
async isMergeableParent(packedParentUid, path) {
|
||||
return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
|
||||
@@ -693,24 +689,22 @@ export class RemoteAdapter {
|
||||
return path
|
||||
}
|
||||
|
||||
async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
|
||||
async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
|
||||
const handler = this._handler
|
||||
|
||||
if (this.useVhdDirectory()) {
|
||||
await writeToVhdDirectory({
|
||||
disk,
|
||||
target: {
|
||||
handler,
|
||||
path,
|
||||
concurrency: writeBlockConcurrency,
|
||||
validator,
|
||||
compression: 'brotli',
|
||||
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
|
||||
const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
|
||||
concurrency: writeBlockConcurrency,
|
||||
compression: this.#getCompressionType(),
|
||||
async validator() {
|
||||
await input.task
|
||||
return validator.apply(this, arguments)
|
||||
},
|
||||
})
|
||||
await VhdAbstract.createAlias(handler, path, dataPath)
|
||||
return size
|
||||
} else {
|
||||
const stream = await toVhdStream(disk)
|
||||
await this.outputStream(path, stream, { validator })
|
||||
await validator(path)
|
||||
return this.outputStream(path, input, { checksum, validator })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -734,16 +728,30 @@ export class RemoteAdapter {
|
||||
}
|
||||
|
||||
// open the hierarchy of ancestors until we find a full one
|
||||
async _createVhdDisk(handler, path, { useChain }) {
|
||||
let disk
|
||||
if (useChain) {
|
||||
disk = await openDiskChain({ handler, path })
|
||||
} else {
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
await disk.init()
|
||||
async _createVhdStream(handler, path, { useChain }) {
|
||||
const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
|
||||
// I don't want the vhds to be disposed on return
|
||||
// but only when the stream is done ( or failed )
|
||||
|
||||
let disposed = false
|
||||
const disposeOnce = async () => {
|
||||
if (!disposed) {
|
||||
disposed = true
|
||||
try {
|
||||
await disposableSynthetic.dispose()
|
||||
} catch (error) {
|
||||
warn('openVhd: failed to dispose VHDs', { error })
|
||||
}
|
||||
}
|
||||
}
|
||||
disk = new ReadAhead(disk)
|
||||
return disk
|
||||
const synthetic = disposableSynthetic.value
|
||||
await synthetic.readBlockAllocationTable()
|
||||
const stream = await synthetic.stream()
|
||||
|
||||
stream.on('end', disposeOnce)
|
||||
stream.on('close', disposeOnce)
|
||||
stream.on('error', disposeOnce)
|
||||
return stream
|
||||
}
|
||||
|
||||
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
|
||||
@@ -751,14 +759,14 @@ export class RemoteAdapter {
|
||||
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
|
||||
const dir = dirname(metadata._filename)
|
||||
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
|
||||
const disks = {}
|
||||
|
||||
const streams = {}
|
||||
await asyncMapSettled(Object.keys(vdis), async ref => {
|
||||
delete vdis[ref].baseVdi
|
||||
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
|
||||
streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
|
||||
})
|
||||
|
||||
return {
|
||||
disks,
|
||||
streams,
|
||||
vbds,
|
||||
vdis,
|
||||
version: '1.0.0',
|
||||
@@ -776,7 +784,7 @@ export class RemoteAdapter {
|
||||
let json
|
||||
let isImmutable = false
|
||||
let remoteIsImmutable = false
|
||||
// if the remote is immutable, check if this metadata is also immutable
|
||||
// if the remote is immutable, check if this metadatas are also immutables
|
||||
try {
|
||||
// this file is not encrypted
|
||||
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
|
||||
@@ -792,7 +800,7 @@ export class RemoteAdapter {
|
||||
json = await this.handler.readFile(path, { flag: 'r+' })
|
||||
// s3 handler don't respect flags
|
||||
} catch (err) {
|
||||
// retry without triggering immutability check ,only on immutable remote
|
||||
// retry without triggerring immutbaility check ,only on immutable remote
|
||||
if (err.code === 'EPERM' && remoteIsImmutable) {
|
||||
isImmutable = true
|
||||
json = await this._handler.readFile(path, { flag: 'r' })
|
||||
|
||||
@@ -24,7 +24,7 @@ export class RestoreMetadataBackup {
|
||||
const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
|
||||
const data = await handler.readFile(dataFileName)
|
||||
|
||||
// if data is JSON, sent it as a plain string; otherwise, consider the data as binary and encode it
|
||||
// if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
|
||||
const isJson = dataFileName.endsWith('.json')
|
||||
return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
|
||||
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
|
||||
// we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
|
||||
})
|
||||
|
||||
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
|
||||
@@ -287,7 +287,7 @@ test('it finish unterminated merge ', async () => {
|
||||
})
|
||||
|
||||
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
|
||||
// the message and resulting files should be identical to the output with vhd files which is tested independently
|
||||
// the message an resulting files should be identical to the output with vhd files which is tested independantly
|
||||
describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
for (const useAlias of [true, false]) {
|
||||
for (const vhdMode of ['file', 'directory']) {
|
||||
@@ -308,9 +308,9 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
|
||||
}
|
||||
|
||||
// a vhd non referenced in metadata
|
||||
// a vhd non referenced in metada
|
||||
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
|
||||
// an abandoned delta vhd without its parent
|
||||
// an abandonded delta vhd without its parent
|
||||
await generateVhd(`${basePath}/abandonned.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
@@ -404,7 +404,7 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
|
||||
|
||||
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
|
||||
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
// grand child and clean vhd should not have changed
|
||||
const survivors = await handler.list(basePath)
|
||||
@@ -454,7 +454,7 @@ test('check Aliases should work alone', async () => {
|
||||
logWarn: () => {},
|
||||
})
|
||||
|
||||
// only ok have survived
|
||||
// only ok have suvived
|
||||
const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
|
||||
assert.equal(alias.length, 1)
|
||||
|
||||
|
||||
@@ -512,7 +512,7 @@ export async function cleanVm(
|
||||
remove,
|
||||
mergeBlockConcurrency,
|
||||
})
|
||||
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
|
||||
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
|
||||
})
|
||||
}
|
||||
@@ -595,7 +595,7 @@ export async function cleanVm(
|
||||
// all disks are now key disk
|
||||
metadata.isVhdDifferencing = {}
|
||||
for (const id of Object.keys(metadata.vdis ?? {})) {
|
||||
metadata.isVhdDifferencing[id] = false
|
||||
metadata.isVhdDifferencing[`${id}.vhd`] = false
|
||||
}
|
||||
}
|
||||
mustRegenerateCache = true
|
||||
|
||||
@@ -26,7 +26,7 @@ const LTR_DEFINITIONS = {
|
||||
|
||||
copy.date(date.date() - firstDayOfWeek)
|
||||
// warning, the year in term of week may different from YYYY
|
||||
// since the computation of the first week of a year is timezone dependent
|
||||
// since the computation of the first week of a year is timezone dependant
|
||||
return copy.format('gggg-ww')
|
||||
}
|
||||
},
|
||||
@@ -59,7 +59,7 @@ const LTR_DEFINITIONS = {
|
||||
* return the entries too old to be kept
|
||||
* if multiple entries are i the same time bucket : keep only the most recent one
|
||||
* if an entry is valid in any of the bucket OR the minRetentionCount : keep it
|
||||
* if a bucket is completely empty : it does not count as one, thus it may extend the retention
|
||||
* if a bucket is completly empty : it does not count as one, thus it may extend the retention
|
||||
* @returns Array<Backup>
|
||||
*/
|
||||
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
|
||||
|
||||
@@ -10,11 +10,6 @@ import { Task } from './Task.mjs'
|
||||
import pick from 'lodash/pick.js'
|
||||
import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
|
||||
|
||||
import { XapiDiskSource } from '@xen-orchestra/xapi'
|
||||
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { toQcow2Stream } from '@xen-orchestra/qcow2'
|
||||
import { VHD_MAX_SIZE } from '@xen-orchestra/xapi/disks/Xapi.mjs'
|
||||
|
||||
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
|
||||
|
||||
export async function exportIncrementalVm(
|
||||
@@ -24,7 +19,7 @@ export async function exportIncrementalVm(
|
||||
) {
|
||||
// refs of VM's VDIs → base's VDIs.
|
||||
|
||||
const disks = {}
|
||||
const streams = {}
|
||||
const vdis = {}
|
||||
const vbds = {}
|
||||
await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
|
||||
@@ -57,14 +52,32 @@ export async function exportIncrementalVm(
|
||||
$snapshot_of$uuid: vdi.$snapshot_of?.uuid,
|
||||
$SR$uuid: vdi.$SR.uuid,
|
||||
}
|
||||
disks[vdiRef] = new XapiDiskSource({
|
||||
vdiRef,
|
||||
xapi: vm.$xapi,
|
||||
baseRef: baseVdi?.$ref,
|
||||
nbdConcurrency,
|
||||
preferNbd,
|
||||
})
|
||||
await disks[vdiRef].init()
|
||||
try {
|
||||
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
|
||||
baseRef: baseVdi?.$ref,
|
||||
cancelToken,
|
||||
format: 'vhd',
|
||||
nbdConcurrency,
|
||||
preferNbd,
|
||||
})
|
||||
} catch (err) {
|
||||
if (err.code === 'VDI_CANT_DO_DELTA') {
|
||||
// fall back to a base
|
||||
Task.info(`Can't do delta, will try to get a full stream`, { vdi })
|
||||
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
|
||||
cancelToken,
|
||||
format: 'vhd',
|
||||
nbdConcurrency,
|
||||
preferNbd,
|
||||
})
|
||||
// only warn if the fall back succeed
|
||||
Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
|
||||
vdi,
|
||||
})
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const suspendVdi = vm.$suspend_VDI
|
||||
@@ -74,13 +87,10 @@ export async function exportIncrementalVm(
|
||||
...suspendVdi,
|
||||
$SR$uuid: suspendVdi.$SR.uuid,
|
||||
}
|
||||
disks[vdiRef] = new XapiDiskSource({
|
||||
vdiRef: suspendVdi.$ref,
|
||||
xapi: vm.$xapi,
|
||||
nbdConcurrency,
|
||||
preferNbd,
|
||||
streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
|
||||
cancelToken,
|
||||
format: 'vhd',
|
||||
})
|
||||
await disks[vdiRef].init()
|
||||
}
|
||||
|
||||
const vifs = {}
|
||||
@@ -106,17 +116,24 @@ export async function exportIncrementalVm(
|
||||
})
|
||||
)
|
||||
|
||||
return {
|
||||
version: '1.1.0',
|
||||
vbds,
|
||||
vdis,
|
||||
vifs,
|
||||
vm: {
|
||||
...vm,
|
||||
return Object.defineProperty(
|
||||
{
|
||||
version: '1.1.0',
|
||||
vbds,
|
||||
vdis,
|
||||
vifs,
|
||||
vm: {
|
||||
...vm,
|
||||
},
|
||||
vtpms,
|
||||
},
|
||||
vtpms,
|
||||
disks,
|
||||
}
|
||||
'streams',
|
||||
{
|
||||
configurable: true,
|
||||
value: streams,
|
||||
writable: true,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
@@ -182,8 +199,8 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
const vdi = vdiRecords[vdiRef]
|
||||
let newVdi
|
||||
|
||||
if (vdi.baseVdi?.$ref !== undefined) {
|
||||
newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
|
||||
if (vdi.baseVdi !== undefined) {
|
||||
newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
|
||||
$defer.onFailure(() => newVdi.$destroy())
|
||||
|
||||
await newVdi.update_other_config(COPY_OF, vdi.uuid)
|
||||
@@ -227,28 +244,21 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
}
|
||||
})
|
||||
|
||||
const { disks } = incrementalVm
|
||||
const { streams } = incrementalVm
|
||||
|
||||
await Promise.all([
|
||||
// Import VDI contents.
|
||||
cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
|
||||
for (const disk of ensureArray(disks[id])) {
|
||||
if (disk === null) {
|
||||
// we restore a backup and reuse completely a local snapshot
|
||||
for (let stream of ensureArray(streams[`${id}.vhd`])) {
|
||||
if (stream === null) {
|
||||
// we restore a backup and reuse completly a local snapshot
|
||||
continue
|
||||
}
|
||||
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
|
||||
|
||||
let stream, format
|
||||
if (vdi.virtual_size > VHD_MAX_SIZE) {
|
||||
stream = await toQcow2Stream(disk)
|
||||
format = 'qcow2'
|
||||
} else {
|
||||
stream = await toVhdStream(disk)
|
||||
format = 'vhd'
|
||||
if (typeof stream === 'function') {
|
||||
stream = await stream()
|
||||
}
|
||||
|
||||
await vdi.$importContent(stream, { cancelToken, format })
|
||||
|
||||
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
|
||||
await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
|
||||
await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -44,7 +44,7 @@ const isValidTar = async (handler, size, fd) => {
|
||||
return buf.every(_ => _ === 0)
|
||||
}
|
||||
|
||||
// TODO: find a heuristic for compressed files
|
||||
// TODO: find an heuristic for compressed files
|
||||
export async function isValidXva(path) {
|
||||
const handler = this._handler
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ async function getDeltaChainLength(xapi, type, ref) {
|
||||
}
|
||||
|
||||
/**
|
||||
* set the delta chain length ( number of delta since last base backup) to a VM and its associated VDIs
|
||||
* set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
@@ -58,7 +58,7 @@ export async function setVmDeltaChainLength(xapi, vmRef, length) {
|
||||
|
||||
/**
|
||||
* Compute the delta chain length of a VM and its associated VDIs
|
||||
* if there is a discrepancy, use, the highest value
|
||||
* if there is a discrependcy, use, the highest value
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
* @returns {Promise}
|
||||
@@ -92,7 +92,7 @@ export function resetVmOtherConfig(xapi, vmRef) {
|
||||
|
||||
/**
|
||||
*
|
||||
* used to ensure compatibility with the previous snapshots that were having the config stored only into VM
|
||||
* used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
@@ -152,7 +152,7 @@ export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, schedule
|
||||
}
|
||||
/**
|
||||
*
|
||||
* mark the export of he VM and its VDIs as successful
|
||||
* mark the export of he VM and its VDIs as successfull
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
|
||||
@@ -4,11 +4,11 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
|
||||
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
|
||||
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
import { FullRemote } from './_vmRunners/FullRemote.mjs'
|
||||
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -41,7 +41,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
const schedule = this._schedule
|
||||
const settings = this._settings
|
||||
|
||||
const throttleGenerator = new Throttle()
|
||||
const throttleStream = createStreamThrottle(settings.maxExportRate)
|
||||
|
||||
const config = this._config
|
||||
|
||||
@@ -89,7 +89,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
sourceRemoteAdapter,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vmUuid,
|
||||
}
|
||||
let vmBackup
|
||||
|
||||
@@ -4,12 +4,12 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
|
||||
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
|
||||
import { runTask } from './_runTask.mjs'
|
||||
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
|
||||
import { FullXapi } from './_vmRunners/FullXapi.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -55,7 +55,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
const schedule = this._schedule
|
||||
const settings = this._settings
|
||||
|
||||
const throttleGenerator = new Throttle()
|
||||
const throttleStream = createStreamThrottle(settings.maxExportRate)
|
||||
|
||||
const config = this._config
|
||||
|
||||
@@ -147,7 +147,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
srs,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}
|
||||
|
||||
|
||||
@@ -24,9 +24,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
||||
writer =>
|
||||
writer.run({
|
||||
stream: forkStreamUnpipe(stream),
|
||||
// stream will be forked and transformed, it's not safe to attach additional properties to it
|
||||
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
|
||||
streamLength: stream.length,
|
||||
maxStreamLength: stream.maxStreamLength, // on encrypted source
|
||||
timestamp: metadata.timestamp,
|
||||
vm: metadata.vm,
|
||||
vmSnapshot: metadata.vmSnapshot,
|
||||
|
||||
@@ -13,35 +13,15 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
return [FullRemoteWriter, FullXapiWriter]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
_mustDoSnapshot() {
|
||||
const vm = this._vm
|
||||
|
||||
const settings = this._settings
|
||||
|
||||
// General setting forcing snapshot at all times
|
||||
if (settings.unconditionalSnapshot) {
|
||||
return true
|
||||
}
|
||||
|
||||
// If offline backup option is not checked
|
||||
// and vm is running, we make a snapshot
|
||||
if (!settings.offlineBackup && vm.power_state === 'Running') {
|
||||
return true
|
||||
}
|
||||
|
||||
// General setting used for snapshot retention
|
||||
// see schedule in backupJob edition
|
||||
if (settings.snapshotRetention !== 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
const vdis = await this._xapi.getRecords('VDI', await vm.$getDisks())
|
||||
const hasNOBAKtag = vdis.some(idx => idx.name_label.includes('[NOBAK]'))
|
||||
|
||||
// if there is a disk with NOBAK tag
|
||||
// we need to make a snapshot
|
||||
if (hasNOBAKtag) {
|
||||
return true
|
||||
}
|
||||
return (
|
||||
settings.unconditionalSnapshot ||
|
||||
(!settings.offlineBackup && vm.power_state === 'Running') ||
|
||||
settings.snapshotRetention !== 0
|
||||
)
|
||||
}
|
||||
_selectBaseVm() {}
|
||||
|
||||
@@ -49,24 +29,22 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
const { compression } = this.job
|
||||
const vm = this._vm
|
||||
const exportedVm = this._exportedVm
|
||||
// @todo put back throttle for full backup/Replication
|
||||
const stream =
|
||||
/* this._throttleStream( */
|
||||
const stream = this._throttleStream(
|
||||
(
|
||||
await this._xapi.VM_export(exportedVm.$ref, {
|
||||
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
|
||||
useSnapshot: false,
|
||||
})
|
||||
).body
|
||||
/* ) */
|
||||
)
|
||||
|
||||
const vdis = await exportedVm.$getDisks()
|
||||
let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe
|
||||
for (const vdiRef of vdis) {
|
||||
const vdi = await this._xapi.getRecord('VDI', vdiRef)
|
||||
|
||||
// the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
|
||||
// of the real stream size in general, since a disk is never completely full
|
||||
// the size a of fully allocated vdi will be virtual_size exaclty, it's a gross over evaluation
|
||||
// of the real stream size in general, since a disk is never completly full
|
||||
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
|
||||
maxStreamLength += vdi.virtual_size
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ import { createLogger } from '@xen-orchestra/log'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import assert from 'node:assert'
|
||||
import * as UUID from 'uuid'
|
||||
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
|
||||
import mapValues from 'lodash/mapValues.js'
|
||||
|
||||
import { AbstractRemote } from './_AbstractRemote.mjs'
|
||||
import { forkDeltaExport } from './_forkDeltaExport.mjs'
|
||||
@@ -10,7 +12,6 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
|
||||
const { warn } = createLogger('xo:backups:Incrementalremote')
|
||||
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
@@ -18,7 +19,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
return IncrementalRemoteWriter
|
||||
}
|
||||
|
||||
// we'll transfer the full list if at least one backup should be transferred
|
||||
// we'll transfer the full list if at least one backup should be transfered
|
||||
// to ensure we don't cut the delta chain
|
||||
_filterTransferList(transferList) {
|
||||
if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
|
||||
@@ -36,7 +37,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
return
|
||||
}
|
||||
await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
|
||||
const isDifferencing = metadata.isVhdDifferencing[id]
|
||||
const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
|
||||
if (isDifferencing) {
|
||||
const vmDir = getVmBackupDir(metadata.vm.uuid)
|
||||
const path = `${vmDir}/${metadata.vhds[id]}`
|
||||
@@ -70,12 +71,9 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
// recompute if disks are differencing or not
|
||||
const isVhdDifferencing = {}
|
||||
|
||||
for (const key in incrementalExport.disks) {
|
||||
const disk = incrementalExport.disks[key]
|
||||
isVhdDifferencing[key] = disk.isDifferencing()
|
||||
incrementalExport.disks[key] = new SynchronizedDisk(disk)
|
||||
}
|
||||
|
||||
await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
|
||||
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
|
||||
})
|
||||
const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
|
||||
if (metadata.isBase === hasDifferencingDisk) {
|
||||
warn(`Metadata isBase and real disk value are different`, {
|
||||
@@ -89,10 +87,11 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
await this._selectBaseVm(metadata)
|
||||
await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
|
||||
|
||||
incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.transfer({
|
||||
deltaExport: forkDeltaExport(incrementalExport, writer.constructor.name),
|
||||
deltaExport: forkDeltaExport(incrementalExport),
|
||||
isVhdDifferencing,
|
||||
timestamp: metadata.timestamp,
|
||||
vm: metadata.vm,
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { pipeline } from 'node:stream'
|
||||
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
|
||||
import keyBy from 'lodash/keyBy.js'
|
||||
import mapValues from 'lodash/mapValues.js'
|
||||
import vhdStreamValidator from 'vhd-lib/vhdStreamValidator.js'
|
||||
|
||||
import { AbstractXapi } from './_AbstractXapi.mjs'
|
||||
import { forkDeltaExport } from './_forkDeltaExport.mjs'
|
||||
import { exportIncrementalVm } from '../../_incrementalVm.mjs'
|
||||
import { forkDeltaExport } from './_forkDeltaExport.mjs'
|
||||
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
|
||||
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import {
|
||||
DATETIME,
|
||||
DELTA_CHAIN_LENGTH,
|
||||
@@ -14,16 +20,16 @@ import {
|
||||
setVmDeltaChainLength,
|
||||
markExportSuccessfull,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
|
||||
const noop = Function.prototype
|
||||
|
||||
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
|
||||
_getWriters() {
|
||||
return [IncrementalRemoteWriter, IncrementalXapiWriter]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
_mustDoSnapshot() {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -39,33 +45,42 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
nbdConcurrency: this._settings.nbdConcurrency,
|
||||
preferNbd: this._settings.preferNbd,
|
||||
})
|
||||
|
||||
const isVhdDifferencing = {}
|
||||
let useNbd = false
|
||||
for (const key in deltaExport.disks) {
|
||||
const disk = deltaExport.disks[key]
|
||||
isVhdDifferencing[key] = disk.isDifferencing()
|
||||
deltaExport.disks[key] = new SynchronizedDisk(disk)
|
||||
useNbd = useNbd || disk.useNbd()
|
||||
}
|
||||
if (useNbd) {
|
||||
// since NBD is network based, if one disk use nbd , all the disk use them
|
||||
// except the suspended VDI
|
||||
if (Object.values(deltaExport.streams).some(({ _nbd }) => _nbd)) {
|
||||
Task.info('Transfer data using NBD')
|
||||
}
|
||||
|
||||
// @todo : reimplement throttle
|
||||
const isVhdDifferencing = {}
|
||||
// since isVhdDifferencingDisk is reading and unshifting data in stream
|
||||
// it should be done BEFORE any other stream transform
|
||||
await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
|
||||
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
|
||||
})
|
||||
const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
|
||||
|
||||
if (this._settings.validateVhdStreams) {
|
||||
deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
|
||||
}
|
||||
deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
|
||||
|
||||
const timestamp = Date.now()
|
||||
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.transfer({
|
||||
deltaExport: forkDeltaExport(deltaExport, writer.constructor.name),
|
||||
deltaExport: forkDeltaExport(deltaExport),
|
||||
isVhdDifferencing,
|
||||
sizeContainers,
|
||||
timestamp,
|
||||
vm,
|
||||
vmSnapshot: exportedVm,
|
||||
}),
|
||||
'writer.transfer()'
|
||||
)
|
||||
|
||||
// we want to control the uuid of the vhd in the chain
|
||||
// and ensure they are correctly chained
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.updateUuidAndChain({
|
||||
@@ -87,6 +102,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
await markExportSuccessfull(this._xapi, exportedVm.$ref)
|
||||
}
|
||||
|
||||
const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
|
||||
const end = Date.now()
|
||||
const duration = end - timestamp
|
||||
debug('transfer complete', {
|
||||
duration,
|
||||
speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
|
||||
size,
|
||||
})
|
||||
|
||||
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
schedule,
|
||||
settings,
|
||||
sourceRemoteAdapter,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vmUuid,
|
||||
}) {
|
||||
super()
|
||||
@@ -34,7 +34,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
|
||||
this._healthCheckSr = healthCheckSr
|
||||
this._sourceRemoteAdapter = sourceRemoteAdapter
|
||||
this._throttleGenerator = throttleGenerator
|
||||
this._throttleStream = throttleStream
|
||||
this._vmUuid = vmUuid
|
||||
|
||||
const allSettings = job.settings
|
||||
@@ -90,7 +90,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
for (const timestamp of timestamps) {
|
||||
if (remoteMetadatas[timestamp] !== nbRemotes) {
|
||||
// this backup is not present in all the remote
|
||||
// should be retransferred if not found later
|
||||
// should be retransfered if not found later
|
||||
transferList.push(localMetada.get(timestamp))
|
||||
} else {
|
||||
// backup is present in local and remote : the chain has already been transferred
|
||||
|
||||
@@ -28,7 +28,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings,
|
||||
srs,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}) {
|
||||
super()
|
||||
@@ -62,7 +62,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
this._healthCheckSr = healthCheckSr
|
||||
this._jobId = job.id
|
||||
this._jobSnapshotVdis = undefined
|
||||
this._throttleGenerator = throttleGenerator
|
||||
this._throttleStream = throttleStream
|
||||
this._xapi = vm.$xapi
|
||||
|
||||
// Base VM for the export
|
||||
@@ -141,14 +141,14 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
|
||||
const settings = this._settings
|
||||
|
||||
if (await this._mustDoSnapshot()) {
|
||||
if (this._mustDoSnapshot()) {
|
||||
await Task.run({ name: 'snapshot' }, async () => {
|
||||
if (!settings.bypassVdiChainsCheck) {
|
||||
await vm.$assertHealthyVdiChains()
|
||||
}
|
||||
if (settings.cbtDestroySnapshotData) {
|
||||
if (settings.preferNbd) {
|
||||
try {
|
||||
// enable CBT on all disks if we want to be able to purge snapshot data
|
||||
// enable CBT on all disks if possible
|
||||
const diskRefs = await xapi.VM_getDisks(vm.$ref)
|
||||
await Promise.all(diskRefs.map(diskRef => xapi.callAsync('VDI.enable_cbt', diskRef)))
|
||||
} catch (error) {
|
||||
@@ -195,7 +195,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
})
|
||||
await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
|
||||
// end of compatibility handling
|
||||
// end of compatibiliy handling
|
||||
|
||||
// handle snapshot by VDI
|
||||
this._jobSnapshotVdis = []
|
||||
@@ -298,7 +298,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
|
||||
// in that case next runs will be full, but there is not an easy way to prevent that
|
||||
this._settings.preferNbd &&
|
||||
// only delete snapshot data if the config allows it
|
||||
// only delete snapshost data if the config allows it
|
||||
this._settings.cbtDestroySnapshotData
|
||||
) {
|
||||
Task.info('will delete snapshot data')
|
||||
@@ -324,7 +324,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
_mustDoSnapshot() {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import cloneDeep from 'lodash/cloneDeep.js'
|
||||
import mapValues from 'lodash/mapValues.js'
|
||||
|
||||
export function forkDeltaExport(deltaExport, label) {
|
||||
label += ' ' + Math.random()
|
||||
const { disks, ...rest } = deltaExport
|
||||
const fork = cloneDeep(rest)
|
||||
fork.disks = {}
|
||||
for (const [key, disk] of Object.entries(disks)) {
|
||||
if (!(disk instanceof SynchronizedDisk)) {
|
||||
throw new Error('Can only fork synchronized disks ')
|
||||
}
|
||||
fork.disks[key] = disk.fork(label)
|
||||
}
|
||||
return fork
|
||||
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
|
||||
|
||||
export function forkDeltaExport(deltaExport) {
|
||||
const { streams, ...rest } = deltaExport
|
||||
const newMetadata = cloneDeep(rest)
|
||||
newMetadata.streams = mapValues(streams, forkStreamUnpipe)
|
||||
return newMetadata
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
const { handler } = this._adapter
|
||||
const vhds = this.#vhds
|
||||
await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
|
||||
const isDifferencing = isVhdDifferencing[id]
|
||||
const isDifferencing = isVhdDifferencing[`${id}.vhd`]
|
||||
const path = `${this._vmBackupDir}/${vhds[id]}`
|
||||
if (isDifferencing) {
|
||||
assert.notStrictEqual(
|
||||
@@ -202,8 +202,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
|
||||
let metadataContent = await this._isAlreadyTransferred(timestamp)
|
||||
if (metadataContent !== undefined) {
|
||||
// skip backup while being vigilant to not stuck the forked disk
|
||||
await Promise.all(Object.values(deltaExport.disks).map(async disk => disk.close()))
|
||||
// skip backup while being vigilant to not stuck the forked stream
|
||||
Task.info('This backup has already been transfered')
|
||||
Object.values(deltaExport.streams).forEach(stream => stream.destroy())
|
||||
return { size: 0 }
|
||||
}
|
||||
|
||||
@@ -222,33 +223,36 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
vmSnapshot,
|
||||
vtpms: deltaExport.vtpms,
|
||||
}
|
||||
let size = 0
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
|
||||
const { size } = await Task.run({ name: 'transfer' }, async () => {
|
||||
let transferSize = 0
|
||||
await asyncEach(
|
||||
Object.entries(deltaExport.disks),
|
||||
async ([diskRef, disk]) => {
|
||||
const path = `${this._vmBackupDir}/${vhds[diskRef]}`
|
||||
await adapter.writeVhd(path, disk, {
|
||||
Object.keys(deltaExport.vdis),
|
||||
async id => {
|
||||
const path = `${this._vmBackupDir}/${vhds[id]}`
|
||||
// don't write it as transferSize += await async function
|
||||
// since i += await asyncFun lead to race condition
|
||||
// as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
|
||||
const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
|
||||
// no checksum for VHDs, because they will be invalidated by
|
||||
// merges and chains
|
||||
// merges and chainings
|
||||
checksum: false,
|
||||
validator: tmpPath => checkVhd(handler, tmpPath),
|
||||
writeBlockConcurrency: this._config.writeBlockConcurrency,
|
||||
})
|
||||
size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
|
||||
transferSize += transferSizeOneDisk
|
||||
},
|
||||
{
|
||||
concurrency: settings.diskPerVmConcurrency,
|
||||
}
|
||||
)
|
||||
|
||||
return { size }
|
||||
return { size: transferSize }
|
||||
})
|
||||
metadataContent.size = size // @todo return exactly the size written by this writer
|
||||
metadataContent.size = size
|
||||
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
|
||||
|
||||
// TODO: run cleanup?
|
||||
return { size }
|
||||
}
|
||||
}
|
||||
decorateClass(IncrementalRemoteWriter, {
|
||||
|
||||
@@ -21,7 +21,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
}
|
||||
|
||||
// @todo use an index if possible
|
||||
// @todo : this seems similar to decorateVmMetadata
|
||||
// @todo : this seems similare to decorateVmMetadata
|
||||
const replicatedVdis = sr.$VDIs
|
||||
.filter(vdi => {
|
||||
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
|
||||
@@ -130,7 +130,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
return backup
|
||||
}
|
||||
|
||||
async _transfer({ timestamp, deltaExport, vm }) {
|
||||
async _transfer({ timestamp, deltaExport, sizeContainers, vm }) {
|
||||
const { _warmMigration } = this._settings
|
||||
const sr = this._sr
|
||||
const job = this._job
|
||||
@@ -141,12 +141,8 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
let targetVmRef
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
|
||||
// size is mandatory to ensure the task have the right data
|
||||
return {
|
||||
size: Object.values(deltaExport.disks).reduce(
|
||||
(sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
|
||||
0
|
||||
),
|
||||
size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
|
||||
}
|
||||
})
|
||||
this._targetVmRef = targetVmRef
|
||||
|
||||
@@ -20,14 +20,10 @@ export class AbstractIncrementalWriter extends AbstractWriter {
|
||||
async transfer({ deltaExport, ...other }) {
|
||||
try {
|
||||
return await this._transfer({ deltaExport, ...other })
|
||||
} catch (err) {
|
||||
console.error({ err })
|
||||
} finally {
|
||||
// ensure all sources are properly closed
|
||||
for (const disk of Object.values(deltaExport.disks)) {
|
||||
try {
|
||||
await disk.close()
|
||||
} catch (err) {}
|
||||
// ensure all streams are properly closed
|
||||
for (const stream of Object.values(deltaExport.streams)) {
|
||||
stream.destroy()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').Disk} Disk
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
*/
|
||||
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { DISK_TYPES } from 'vhd-lib/_constants.js'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
/**
|
||||
* Represents a remote VHD (Virtual Hard Disk) that extends RandomAccessDisk.
|
||||
*/
|
||||
export class RemoteVhd extends RandomAccessDisk {
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @type {VhdFile | VhdDirectory | undefined}
|
||||
*/
|
||||
#vhd
|
||||
|
||||
/**
|
||||
* @type {boolean | undefined}
|
||||
*/
|
||||
#isDifferencing
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
get path() {
|
||||
return this.#path
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getvirtualsize of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.footer.currentSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return 2 * 1024 * 1024
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init() {
|
||||
const { value, dispose } = await openVhd(this.#handler, this.#path)
|
||||
this.#vhd = value
|
||||
this.#dispose = dispose
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await this.#dispose()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call hasblock of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.containsBlock(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getBlockIndexes of a RemoteVhd before init`)
|
||||
}
|
||||
const index = []
|
||||
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
|
||||
if (this.hasBlock(blockIndex)) {
|
||||
index.push(blockIndex)
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>}
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhd before init`)
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @returns {RandomAccessDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call openParent of a RemoteVhd before init`)
|
||||
}
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
if (!parentPath) {
|
||||
throw new Error(`Disk ${this.#path} doesn't have parents`)
|
||||
}
|
||||
const parent = new RemoteVhd({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
if (this.#isDifferencing === undefined) {
|
||||
throw new Error(`can't call isDifferencing of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#isDifferencing
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
// @ts-check
|
||||
/**
|
||||
*
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
*/
|
||||
import { DiskChain } from '@xen-orchestra/disk-transform'
|
||||
import { RemoteVhd } from './RemoteVhd.mjs'
|
||||
|
||||
import { defer } from 'golike-defer'
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {string | undefined} params.until
|
||||
*/
|
||||
async function _openDiskChain($defer, { handler, path, until }) {
|
||||
let disk
|
||||
const disks = []
|
||||
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
|
||||
await disk.init()
|
||||
disks.push(disk)
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent()
|
||||
if (disk.path === until) {
|
||||
break
|
||||
}
|
||||
disks.unshift(disk)
|
||||
}
|
||||
// the root disk
|
||||
return new DiskChain({ disks })
|
||||
}
|
||||
|
||||
export const openDiskChain = defer(_openDiskChain)
|
||||
@@ -53,7 +53,7 @@ Add the following file: `xo-vm-backups/<VM UUID>/cache.json.gz`.
|
||||
|
||||
This cache is compressed in Gzip and contains an JSON object with the metadata for all the backups of this VM indexed by their absolute path (i.e. `/xo-vm-backups/<VM UUID>/<timestamp>.json`).
|
||||
|
||||
This file is generated on demand when listing the backups, and directly updated on backup creation/deletion.
|
||||
This file is generated on demande when listing the backups, and directly updated on backup creation/deletion.
|
||||
|
||||
In case any incoherence is detected, the file is deleted so it will be fully generated when required.
|
||||
|
||||
@@ -180,7 +180,7 @@ job:
|
||||
sourceRemote: IdPattern
|
||||
|
||||
# Indicates which XAPI to use to connect to a specific VM or SR
|
||||
# for remote to remote backup job,this is only needed if there is healthcheck
|
||||
# for remote to remote backup job,this is only needed if there is healtcheck
|
||||
recordToXapi:
|
||||
[ObjectId]: XapiId
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.63.0",
|
||||
"version": "0.59.0",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
},
|
||||
@@ -25,14 +25,11 @@
|
||||
"@vates/decorate-with": "^2.1.0",
|
||||
"@vates/disposable": "^0.1.6",
|
||||
"@vates/fuse-vhd": "^2.1.2",
|
||||
"@vates/generator-toolbox": "^1.0.4",
|
||||
"@vates/nbd-client": "^3.1.4",
|
||||
"@vates/nbd-client": "^3.1.2",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/disk-transform": "^1.1.0",
|
||||
"@xen-orchestra/fs": "^4.6.2",
|
||||
"@xen-orchestra/fs": "^4.5.0",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/qcow2": "^1.0.0",
|
||||
"@xen-orchestra/template": "^0.1.0",
|
||||
"app-conf": "^3.0.0",
|
||||
"compare-versions": "^6.0.0",
|
||||
@@ -51,8 +48,8 @@
|
||||
"tar": "^6.1.15",
|
||||
"uuid": "^9.0.0",
|
||||
"value-matcher": "^0.2.0",
|
||||
"vhd-lib": "^4.14.0",
|
||||
"xen-api": "^4.7.3",
|
||||
"vhd-lib": "^4.11.3",
|
||||
"xen-api": "^4.7.1",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -62,7 +59,7 @@
|
||||
"tmp": "^0.2.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^8.3.3"
|
||||
"@xen-orchestra/xapi": "^8.1.1"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"preferGlobal": true,
|
||||
"dependencies": {
|
||||
"golike-defer": "^0.5.1",
|
||||
"xen-api": "^4.7.3"
|
||||
"xen-api": "^4.7.1"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish"
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../../scripts/npmignore
|
||||
@@ -1,6 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
import eslint from '@eslint/js'
|
||||
import tseslint from 'typescript-eslint'
|
||||
|
||||
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended)
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"name": "@xen-orchestra/disk-transform",
|
||||
"version": "1.1.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"license": "MIT",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"rimraf": "^5.0.1",
|
||||
"@types/node": "^22.3.0",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.23.0"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "tsc && node --test **/*.test.mjs",
|
||||
"clean": "rimraf dist/"
|
||||
},
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/disk-transform",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@xen-orchestra/disk-transform",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.17"
|
||||
},
|
||||
"dependencies": {
|
||||
"vhd-lib": "^4.14.0",
|
||||
"@vates/generator-toolbox": "^1.0.4"
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
import { ProgressHandler } from './ProgressHandler.mjs'
|
||||
|
||||
export type DiskBlockData = Buffer
|
||||
export type DiskBlock = {
|
||||
index: number // the index of the block. Offset in raw disk is index * blockSize
|
||||
data: DiskBlockData // thue Buffer like data of this block. Must be blockSize length
|
||||
}
|
||||
|
||||
export type BytesLength = number
|
||||
|
||||
export abstract class Disk {
|
||||
#generatedDiskBlocks = 0
|
||||
#parent?: Disk
|
||||
progressHandler?: ProgressHandler
|
||||
get parent(): Disk | undefined {
|
||||
return this.#parent
|
||||
}
|
||||
|
||||
abstract getVirtualSize(): number
|
||||
abstract getBlockSize(): number
|
||||
abstract init(): Promise<void>
|
||||
abstract close(): Promise<void>
|
||||
|
||||
abstract isDifferencing(): boolean
|
||||
// optional method
|
||||
instantiateParent(): Disk {
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
async openParent(): Promise<Disk> {
|
||||
if (this.#parent !== undefined) {
|
||||
return this.#parent
|
||||
}
|
||||
if (!this.isDifferencing()) {
|
||||
throw new Error("Can't open the parent of a non differencing disk")
|
||||
}
|
||||
this.#parent = this.instantiateParent()
|
||||
await this.#parent.init()
|
||||
return this.#parent
|
||||
}
|
||||
|
||||
/**
|
||||
* return the block without any order nor stability guarantee
|
||||
*/
|
||||
abstract getBlockIndexes(): Array<number>
|
||||
// must return true if the block is present in this disk
|
||||
// without looking at the parent
|
||||
abstract hasBlock(index: number): boolean
|
||||
abstract buildDiskBlockGenerator(): Promise<AsyncGenerator<DiskBlock>> | AsyncGenerator<DiskBlock>
|
||||
async *diskBlocks(uid?: string): AsyncGenerator<DiskBlock> {
|
||||
try {
|
||||
// compute next block while the destination is consuming the current block
|
||||
const blockGenerator = await this.buildDiskBlockGenerator()
|
||||
let next = blockGenerator.next()
|
||||
while (true) {
|
||||
const res = await next
|
||||
next = blockGenerator.next()
|
||||
if (res.done) {
|
||||
break
|
||||
}
|
||||
this.#generatedDiskBlocks++
|
||||
yield res.value
|
||||
}
|
||||
} finally {
|
||||
await this.close()
|
||||
}
|
||||
}
|
||||
|
||||
getNbGeneratedBlock(): number {
|
||||
return this.#generatedDiskBlocks
|
||||
}
|
||||
check() {
|
||||
// @todo move here the checks done in cleanVm if possible
|
||||
return Promise.resolve(true)
|
||||
}
|
||||
|
||||
// @todo readRawData will be needed for file level restore
|
||||
// @todo rename will be needed for merge
|
||||
// @todo : should the alias part moved from remote/vhd ?
|
||||
}
|
||||
|
||||
/**
|
||||
* Disks implementing this class can be accessed in any order
|
||||
* For example xva will need to have data in the offset order
|
||||
*/
|
||||
|
||||
export abstract class RandomAccessDisk extends Disk {
|
||||
get parent(): RandomAccessDisk | undefined {
|
||||
if (super.parent !== undefined && !(super.parent instanceof RandomAccessDisk)) {
|
||||
throw new Error('The parent of a random access disk must be a random access disk')
|
||||
}
|
||||
return super.parent
|
||||
}
|
||||
// optional method
|
||||
instantiateParent(): RandomAccessDisk {
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
// can read data parent if block size are not aligned
|
||||
// but only if this disk has data on this block
|
||||
abstract readBlock(index: number): Promise<DiskBlock>
|
||||
async *buildDiskBlockGenerator(): AsyncGenerator<DiskBlock> {
|
||||
try {
|
||||
const indexes = this.getBlockIndexes()
|
||||
for (let i = 0; i < indexes.length; i++) {
|
||||
yield this.readBlock(indexes[i])
|
||||
await this.progressHandler?.setProgress(i / indexes.length)
|
||||
}
|
||||
await this.progressHandler?.setProgress(1)
|
||||
} finally {
|
||||
await this.progressHandler?.done()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
import { afterEach, beforeEach, test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import { DebugDisk } from './utils/DebugDisk.mjs'
|
||||
|
||||
test('Disk class', async t => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 20
|
||||
let disk: DebugDisk
|
||||
beforeEach(async () => {
|
||||
disk = new DebugDisk({ nbBlocks, blockSize, fillRate: 0 })
|
||||
|
||||
await disk.init()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await disk.close()
|
||||
})
|
||||
await t.test('getVirtualSize', () => {
|
||||
assert.strictEqual(disk.getVirtualSize(), blockSize * nbBlocks)
|
||||
})
|
||||
|
||||
await t.test('getBlockSize', () => {
|
||||
assert.strictEqual(disk.getBlockSize(), blockSize)
|
||||
})
|
||||
|
||||
await t.test('diskBlocks generator', async () => {
|
||||
const keys = [4, 15]
|
||||
for (const key of keys) {
|
||||
disk.blocks[key] = true
|
||||
}
|
||||
const foundKeys = []
|
||||
const generator = disk.diskBlocks()
|
||||
for await (const block of generator) {
|
||||
foundKeys.push(block.index)
|
||||
assert.strictEqual(block.data.length, blockSize)
|
||||
}
|
||||
assert.strictEqual(disk.getNbGeneratedBlock(), 2)
|
||||
|
||||
keys.sort()
|
||||
foundKeys.sort()
|
||||
assert.deepStrictEqual(keys, foundKeys)
|
||||
})
|
||||
})
|
||||
@@ -1,69 +0,0 @@
|
||||
import { Disk, RandomAccessDisk, type DiskBlock } from './Disk.mjs'
|
||||
|
||||
export class DiskChain extends RandomAccessDisk {
|
||||
#disks: Array<RandomAccessDisk> = []
|
||||
|
||||
constructor({ disks }: { disks: Array<RandomAccessDisk> }) {
|
||||
super()
|
||||
this.#disks = disks
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
return this.#disks[this.#disks.length - 1].getVirtualSize()
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.#disks[0].getBlockSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* the main difference with the base disk block method
|
||||
* is that if any of the disk has this block it return true
|
||||
*
|
||||
*/
|
||||
hasBlock(index: number): boolean {
|
||||
for (let i = this.#disks.length - 1; i >= 0; i--) {
|
||||
if (this.#disks[i].hasBlock(index)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Read in the last disk first an only go back if the block is not present
|
||||
* @param index
|
||||
* @returns
|
||||
*/
|
||||
readBlock(index: number): Promise<DiskBlock> {
|
||||
for (let i = this.#disks.length - 1; i >= 0; i--) {
|
||||
if (this.#disks[i].hasBlock(index)) {
|
||||
return this.#disks[i].readBlock(index)
|
||||
}
|
||||
}
|
||||
throw new Error(`Block ${index} not found in chain `)
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
await Promise.all(this.#disks.map(disk => disk.init()))
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
await Promise.all(this.#disks.map(disk => disk.close()))
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
return [...new Set(this.#disks.map(disk => disk.getBlockIndexes()).flat())]
|
||||
}
|
||||
|
||||
isDifferencing(): boolean {
|
||||
return this.#disks[0].isDifferencing()
|
||||
}
|
||||
static async openFromChild(child:RandomAccessDisk):Promise<RandomAccessDisk>{
|
||||
let disk = child
|
||||
const disks=[disk]
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent() as RandomAccessDisk
|
||||
disks.unshift(disk)
|
||||
// @todo handle until
|
||||
}
|
||||
return new DiskChain({ disks })
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import { afterEach, beforeEach, test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import { DebugDisk } from './utils/DebugDisk.mjs'
|
||||
import { DiskChain } from './DiskChain.mjs'
|
||||
|
||||
test('Disk Chain class', async t => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 20
|
||||
const nbDisks = 5
|
||||
const disks: Array<DebugDisk> = []
|
||||
let chain: DiskChain
|
||||
beforeEach(async () => {
|
||||
for (let i = 0; i < nbDisks; i++) {
|
||||
const disk = new DebugDisk({ nbBlocks: nbBlocks + 1, blockSize, fillRate: 0, filledBy: i })
|
||||
disks.push(disk)
|
||||
}
|
||||
chain = new DiskChain({ disks })
|
||||
await chain.init()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await chain.close()
|
||||
})
|
||||
await t.test('getVirtualSize', () => {
|
||||
// the size of the last one
|
||||
assert.strictEqual(chain.getVirtualSize(), disks[disks.length - 1].getVirtualSize())
|
||||
})
|
||||
|
||||
await t.test('getBlockSize', () => {
|
||||
assert.strictEqual(chain.getBlockSize(), blockSize)
|
||||
})
|
||||
|
||||
await t.test('diskBlocks generator read from the right disk', async () => {
|
||||
const keys = []
|
||||
|
||||
for (let blockIndex = 0; blockIndex < nbBlocks; blockIndex++) {
|
||||
disks[0].blocks[blockIndex] = true //each disk only have one block, its disk index
|
||||
keys.push(blockIndex)
|
||||
}
|
||||
for (let diskIndex = 0; diskIndex < nbDisks; diskIndex++) {
|
||||
disks[diskIndex].blocks[diskIndex] = true //each disk only have one block, its disk index
|
||||
}
|
||||
|
||||
const foundKeys = []
|
||||
const generator = chain.diskBlocks()
|
||||
for await (const block of generator) {
|
||||
foundKeys.push(block.index)
|
||||
assert.strictEqual(block.data.length, blockSize)
|
||||
if (block.index < nbDisks) {
|
||||
assert.strictEqual(block.data.readUInt8(0), block.index)
|
||||
} else {
|
||||
// read from first
|
||||
assert.strictEqual(block.data.readUInt8(0), 0)
|
||||
}
|
||||
}
|
||||
|
||||
assert.strictEqual(chain.getNbGeneratedBlock(), nbBlocks)
|
||||
|
||||
keys.sort()
|
||||
foundKeys.sort()
|
||||
assert.deepStrictEqual(keys, foundKeys)
|
||||
})
|
||||
})
|
||||
@@ -1,106 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { DiskBlock, RandomAccessDisk } from './Disk.mjs'
|
||||
import { RandomDiskPassthrough } from './DiskPassthrough.mjs'
|
||||
import { DiskChain } from './DiskChain.mjs'
|
||||
|
||||
export class DiskLargerBlock extends RandomAccessDisk {
|
||||
#blockSize
|
||||
#source: RandomAccessDisk
|
||||
#parent?: RandomAccessDisk
|
||||
get source(): RandomAccessDisk {
|
||||
return this.#source
|
||||
}
|
||||
|
||||
constructor(source: RandomAccessDisk, blockSize: number) {
|
||||
super()
|
||||
this.#source = source
|
||||
assert.ok(
|
||||
blockSize >= source.getBlockSize(),
|
||||
`target block size ${blockSize} must be bigger than the source block size ${source.getBlockSize()} `
|
||||
)
|
||||
|
||||
assert.strictEqual(
|
||||
blockSize % source.getBlockSize(),
|
||||
0,
|
||||
`target block size ${blockSize} must be a multiple of the source block size ${source.getBlockSize()} `
|
||||
)
|
||||
this.#blockSize = blockSize
|
||||
}
|
||||
openSource(): Promise<RandomAccessDisk> {
|
||||
// not an issue since source MUST BE passed to the constructor
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
// @todo handle partial block at the end
|
||||
const source = this.source
|
||||
const destinationBlock = Buffer.alloc(this.getBlockSize(), 0)
|
||||
const blockRatio = this.#blockSize / source.getBlockSize()
|
||||
for (let i = index * blockRatio; i < (index + 1) * blockRatio; i++) {
|
||||
let data: Buffer | undefined
|
||||
if (source.hasBlock(i)) {
|
||||
data = (await source.readBlock(i)).data
|
||||
} else {
|
||||
if (this.isDifferencing()) {
|
||||
if (this.#parent === undefined) {
|
||||
const directParent = (await source.openParent()) as RandomAccessDisk
|
||||
const chain = await DiskChain.openFromChild(directParent)
|
||||
this.#parent = chain
|
||||
}
|
||||
const parent = this.#parent!
|
||||
data = (await parent.readBlock(i)).data
|
||||
}
|
||||
}
|
||||
if (data !== undefined) {
|
||||
data.copy(destinationBlock, i * source.getBlockSize())
|
||||
}
|
||||
}
|
||||
return {
|
||||
index,
|
||||
data: destinationBlock,
|
||||
}
|
||||
}
|
||||
|
||||
getBlockSize(): number {
|
||||
return this.#blockSize
|
||||
}
|
||||
|
||||
getBlockIndexes(): Array<number> {
|
||||
const maxBlock = Math.ceil(this.getVirtualSize() / this.getBlockSize())
|
||||
const blockRatio = this.getBlockSize() / this.source.getBlockSize()
|
||||
const indexes = []
|
||||
for (let i = 0; i < maxBlock; i++) {
|
||||
for (let j = 0; j < blockRatio; j++) {
|
||||
if (this.source.hasBlock(i * blockRatio + j)) {
|
||||
indexes.push(i)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
|
||||
hasBlock(index: number): boolean {
|
||||
const source = this.source
|
||||
let maxBlockSource = Math.ceil(source.getVirtualSize() / source.getBlockSize())
|
||||
let blockRatio = this.#blockSize / this.source.getBlockSize()
|
||||
for (let i = index * blockRatio; i < Math.min((index + 1) * blockRatio, maxBlockSource); i++) {
|
||||
if (this.source.hasBlock(i)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
getVirtualSize(): number {
|
||||
return this.source.getVirtualSize()
|
||||
}
|
||||
init(): Promise<void> {
|
||||
return this.source.init()
|
||||
}
|
||||
close(): Promise<void> {
|
||||
return this.source.close()
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return this.source.isDifferencing()
|
||||
}
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { test } from 'node:test'
|
||||
import { DiskLargerBlock } from './DiskLargerBlock.mjs'
|
||||
import { RandomAccessDisk, DiskBlock, Disk } from './Disk.mjs'
|
||||
|
||||
// Mock implementation of RandomAccessDisk for testing
|
||||
class MockDisk extends RandomAccessDisk {
|
||||
private blockSize: number
|
||||
private virtualSize: number
|
||||
private blocks: Map<number, Buffer>
|
||||
public closed: boolean = false
|
||||
public initialized: boolean = false
|
||||
public isDiff: boolean = false
|
||||
public parentDisk: RandomAccessDisk | null = null
|
||||
|
||||
constructor(blockSize: number, virtualSize: number, blocks: [number, Buffer][] = []) {
|
||||
super()
|
||||
this.blockSize = blockSize
|
||||
this.virtualSize = virtualSize
|
||||
this.blocks = new Map(blocks)
|
||||
}
|
||||
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
if (this.closed) throw new Error('Disk closed')
|
||||
if (!this.initialized) throw new Error('Disk not initialized')
|
||||
if (!this.blocks.has(index)) {
|
||||
if (this.isDiff && this.parentDisk) {
|
||||
return this.parentDisk.readBlock(index)
|
||||
}
|
||||
throw new Error(`Block ${index} not found`)
|
||||
}
|
||||
return { index, data: this.blocks.get(index)! }
|
||||
}
|
||||
|
||||
getVirtualSize(): number {
|
||||
return this.virtualSize
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.blockSize
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
this.initialized = true
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
this.closed = true
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return this.isDiff
|
||||
}
|
||||
instantiateParent(): RandomAccessDisk {
|
||||
if (!this.parentDisk) throw new Error('No parent disk available')
|
||||
return this.parentDisk
|
||||
}
|
||||
getBlockIndexes(): number[] {
|
||||
return [...this.blocks.keys()]
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.blocks.has(index) || (this.isDiff && !!this.parentDisk && this.parentDisk.hasBlock(index))
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create a buffer with pattern
|
||||
function createPatternBuffer(size: number, pattern: string): Buffer {
|
||||
const buf = Buffer.alloc(size)
|
||||
for (let i = 0; i < size; i++) {
|
||||
buf[i] = pattern.charCodeAt(i % pattern.length)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
// Test suite
|
||||
test('constructor and initialization', async () => {
|
||||
const source = new MockDisk(512, 1024 * 1024)
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
|
||||
// Test that blockSize must be a multiple of source blockSize
|
||||
await assert.rejects(
|
||||
async () => {
|
||||
const invalidDisk = new DiskLargerBlock(source, 513)
|
||||
await invalidDisk.init()
|
||||
},
|
||||
{ message: /must be a multiple/ }
|
||||
)
|
||||
// Test that blockSize must be bigger than source blockSize
|
||||
await assert.rejects(
|
||||
async () => {
|
||||
const invalidDisk = new DiskLargerBlock(source, 128)
|
||||
await invalidDisk.init()
|
||||
},
|
||||
{ message: /bigger/ }
|
||||
)
|
||||
|
||||
await disk.init()
|
||||
assert.strictEqual(disk.getBlockSize(), 1024)
|
||||
assert.strictEqual(disk.getVirtualSize(), 1024 * 1024)
|
||||
})
|
||||
|
||||
test('readBlock with simple block mapping', async () => {
|
||||
// Create source disk with two 512-byte blocks
|
||||
const block1 = createPatternBuffer(512, 'A')
|
||||
const block2 = createPatternBuffer(512, 'B')
|
||||
const source = new MockDisk(512, 1024, [
|
||||
[0, block1],
|
||||
[1, block2],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
|
||||
// Read the combined block (should be block1 + block2)
|
||||
const result = await disk.readBlock(0)
|
||||
assert.strictEqual(result.index, 0)
|
||||
assert.strictEqual(result.data.length, 1024)
|
||||
|
||||
// Verify the data is correctly combined
|
||||
assert(result.data.subarray(0, 512).equals(block1))
|
||||
assert(result.data.subarray(512).equals(block2))
|
||||
})
|
||||
|
||||
test('hasBlock behavior', async () => {
|
||||
const source = new MockDisk(512, 2048, [
|
||||
[0, Buffer.alloc(512)],
|
||||
[2, Buffer.alloc(512)],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
|
||||
// Block 0 in disk (1024) covers blocks 0-1 in source (512)
|
||||
// Only block 0 exists in source, so hasBlock(0) should return true
|
||||
assert.strictEqual(disk.hasBlock(0), true)
|
||||
|
||||
// Block 1 in disk covers blocks 2-3 in source
|
||||
// Block 2 exists but 3 doesn't, hasBlock(1) should still return true
|
||||
assert.strictEqual(disk.hasBlock(1), true)
|
||||
|
||||
// Block 2 in disk would cover blocks 4-5 which don't exist
|
||||
assert.strictEqual(disk.hasBlock(2), false)
|
||||
})
|
||||
|
||||
test('getBlockIndexes', async () => {
|
||||
const source = new MockDisk(512, 2048, [
|
||||
[0, Buffer.alloc(512)],
|
||||
[2, Buffer.alloc(512)],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
|
||||
// Should return indexes where at least one source block exists in the range
|
||||
const indexes = disk.getBlockIndexes()
|
||||
assert.deepStrictEqual(indexes, [0, 1])
|
||||
})
|
||||
|
||||
test('partial block handling', async () => {
|
||||
// Test when the last block is partial
|
||||
const source = new MockDisk(512, 1536, [
|
||||
[0, Buffer.alloc(512)],
|
||||
[1, Buffer.alloc(512)],
|
||||
[2, Buffer.alloc(512)],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
|
||||
// First block is complete (blocks 0-1)
|
||||
const block0 = await disk.readBlock(0)
|
||||
assert.strictEqual(block0.data.length, 1024)
|
||||
|
||||
// Second block is partial (only block 2 exists)
|
||||
const block1 = await disk.readBlock(1)
|
||||
assert.strictEqual(block1.data.length, 1024)
|
||||
|
||||
// The first 512 bytes should be from block 2
|
||||
// The remaining 512 bytes should be zeros (since we allocate with zeros)
|
||||
assert(block1.data.subarray(512).every(b => b === 0))
|
||||
assert(block1.data.subarray(0, 512).equals((await source.readBlock(2)).data))
|
||||
})
|
||||
|
||||
test('differencing disk behavior', async () => {
|
||||
// Create parent disk
|
||||
const parentBlock = createPatternBuffer(512, 'PARENT')
|
||||
const parent = new MockDisk(512, 1024, [[0, parentBlock]])
|
||||
await parent.init()
|
||||
|
||||
// Create differencing disk
|
||||
const diffBlock = createPatternBuffer(512, 'DIFF')
|
||||
const diff = new MockDisk(512, 1024, [[1, diffBlock]])
|
||||
diff.isDiff = true
|
||||
diff.parentDisk = parent
|
||||
await diff.init()
|
||||
|
||||
const disk = new DiskLargerBlock(diff, 1024)
|
||||
await disk.init()
|
||||
|
||||
// Block 0 should combine parent block 0 and diff block 1
|
||||
const result = await disk.readBlock(0)
|
||||
assert.strictEqual(result.data.length, 1024)
|
||||
assert(result.data.subarray(0, 512).equals(parentBlock))
|
||||
assert(result.data.subarray(512).equals(diffBlock))
|
||||
})
|
||||
|
||||
test('close propagation', async () => {
|
||||
const source = new MockDisk(512, 1024)
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
|
||||
await disk.close()
|
||||
assert.strictEqual(source.closed, true)
|
||||
})
|
||||
|
||||
test('generator', async () => {
|
||||
const source = new MockDisk(512, 1024)
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskLargerBlock(source, 1024)
|
||||
await disk.init()
|
||||
for await (const block of disk.diskBlocks()) {
|
||||
assert.strictEqual(block.data.length, 1024)
|
||||
}
|
||||
})
|
||||
@@ -1,108 +0,0 @@
|
||||
import { Disk, DiskBlock, RandomAccessDisk } from './Disk.mjs'
|
||||
|
||||
export class DiskPassthrough extends Disk {
|
||||
#source: Disk | undefined
|
||||
get source(): Disk {
|
||||
if (this.#source === undefined) {
|
||||
throw new Error(`Either transmit the source to the constructor or implement openSource and call init`)
|
||||
}
|
||||
return this.#source
|
||||
}
|
||||
get parent(): Disk | undefined {
|
||||
return this.#source?.parent
|
||||
}
|
||||
constructor(source: Disk | undefined = undefined) {
|
||||
super()
|
||||
this.#source = source
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
return this.source.getVirtualSize()
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.source.getBlockSize()
|
||||
}
|
||||
|
||||
async openSource(): Promise<Disk> {
|
||||
throw new Error('open source should be implemented to handle complex open scenario')
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
// open only if nothing has been given to the constructor
|
||||
this.#source = this.#source ?? (await this.openSource())
|
||||
}
|
||||
|
||||
instantiateParent(): Disk {
|
||||
return this.source.instantiateParent()
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
await this.#source?.close()
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return this.source.isDifferencing()
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
return this.source.getBlockIndexes()
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.source.hasBlock(index)
|
||||
}
|
||||
async buildDiskBlockGenerator(): Promise<AsyncGenerator<DiskBlock>> {
|
||||
return this.source.buildDiskBlockGenerator()
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class RandomDiskPassthrough extends RandomAccessDisk {
|
||||
#source: RandomAccessDisk | undefined
|
||||
get source(): RandomAccessDisk {
|
||||
if (this.#source === undefined) {
|
||||
throw new Error(`Either transmit the source to the constructor or implement openSource and call init`)
|
||||
}
|
||||
return this.#source
|
||||
}
|
||||
|
||||
get parent(): RandomAccessDisk | undefined {
|
||||
return this.#source?.parent as RandomAccessDisk
|
||||
}
|
||||
|
||||
constructor(source: RandomAccessDisk | undefined) {
|
||||
super()
|
||||
this.#source = source
|
||||
}
|
||||
/**
|
||||
* return an empty block if asking for and block not included in this disk
|
||||
* @param index
|
||||
* @returns {Promise<DiskBlock>}
|
||||
*/
|
||||
readBlock(index: number): Promise<DiskBlock> {
|
||||
return this.source.readBlock(index)
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
return this.source.getVirtualSize()
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.source.getBlockSize()
|
||||
}
|
||||
|
||||
abstract openSource(): Promise<RandomAccessDisk>
|
||||
async init(): Promise<void> {
|
||||
// open only if nothing has been given to the constructor
|
||||
this.#source = this.#source ?? (await this.openSource())
|
||||
}
|
||||
|
||||
instantiateParent(): RandomAccessDisk {
|
||||
return this.source.instantiateParent()
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.source?.close()
|
||||
}
|
||||
|
||||
isDifferencing(): boolean {
|
||||
return this.source.isDifferencing()
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
return this.source.getBlockIndexes()
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.source.hasBlock(index)
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
import { afterEach, beforeEach, test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import { DebugDisk } from './utils/DebugDisk.mjs'
|
||||
import { DiskPassthrough } from './DiskPassthrough.mjs'
|
||||
import { consume } from './utils/DebugConsumer.mjs'
|
||||
|
||||
test('Disk Passthrough', async t => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 20
|
||||
let disk: DebugDisk
|
||||
beforeEach(async () => {
|
||||
disk = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })
|
||||
|
||||
await disk.init()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await disk.close()
|
||||
})
|
||||
await t.test('source in constructor', async () => {
|
||||
const passthrough = new DiskPassthrough(disk)
|
||||
await assert.rejects(() => passthrough.openSource())
|
||||
assert.strictEqual(passthrough.getBlockSize(), blockSize)
|
||||
assert.strictEqual(passthrough.getVirtualSize(), blockSize * nbBlocks)
|
||||
await consume(passthrough)
|
||||
assert.strictEqual(passthrough.getNbGeneratedBlock(), nbBlocks)
|
||||
assert.strictEqual(disk.closeDone, true)
|
||||
})
|
||||
|
||||
await t.test('source open async', async () => {
|
||||
class DebugPT extends DiskPassthrough {
|
||||
async openSource() {
|
||||
return Promise.resolve(disk)
|
||||
}
|
||||
}
|
||||
const passthrough = new DebugPT()
|
||||
assert.throws(() => passthrough.source)
|
||||
await passthrough.init()
|
||||
assert.strictEqual(passthrough.getBlockSize(), blockSize)
|
||||
assert.strictEqual(passthrough.getVirtualSize(), blockSize * nbBlocks)
|
||||
await consume(passthrough)
|
||||
assert.strictEqual(passthrough.getNbGeneratedBlock(), nbBlocks)
|
||||
assert.strictEqual(disk.closeDone, true)
|
||||
})
|
||||
})
|
||||
@@ -1,99 +0,0 @@
|
||||
import assert, { throws } from 'node:assert'
|
||||
import { RandomAccessDisk, type Disk, type DiskBlock } from './Disk.mjs'
|
||||
import { DiskPassthrough } from './DiskPassthrough.mjs'
|
||||
|
||||
export class DiskSmallerBlock extends DiskPassthrough {
|
||||
#blockSize
|
||||
#generatedDiskBlocks = 0
|
||||
#blockCache?: DiskBlock
|
||||
|
||||
constructor(source: Disk, blockSize: number) {
|
||||
super(source)
|
||||
assert.ok(
|
||||
blockSize <= source.getBlockSize(),
|
||||
`target block size ${blockSize} must be smaller than the source block size ${source.getBlockSize()} `
|
||||
)
|
||||
|
||||
assert.strictEqual(
|
||||
source.getBlockSize() % blockSize,
|
||||
0,
|
||||
`target block size ${blockSize} must be a multiple of the source block size ${source.getBlockSize()} `
|
||||
)
|
||||
this.#blockSize = blockSize
|
||||
}
|
||||
|
||||
openSource(): Promise<Disk> {
|
||||
// not an issue since source MUST BE passed to the constructor
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
|
||||
getNbGeneratedBlock(): number {
|
||||
return this.#generatedDiskBlocks
|
||||
}
|
||||
|
||||
async *diskBlocks(): AsyncGenerator<DiskBlock> {
|
||||
try {
|
||||
const blockRatio = this.source.getBlockSize() / this.#blockSize
|
||||
const sourceGenerator = this.source.diskBlocks()
|
||||
for await (const { data: sourceData, index: sourceIndex } of sourceGenerator) {
|
||||
for (let i = 0; i < blockRatio; i++) {
|
||||
this.#generatedDiskBlocks++
|
||||
const data = sourceData.subarray(i * this.#blockSize, (i + 1) * this.#blockSize)
|
||||
yield {
|
||||
index: sourceIndex * blockRatio + i,
|
||||
data,
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await this.progressHandler?.done()
|
||||
await this.close()
|
||||
}
|
||||
}
|
||||
|
||||
getBlockSize(): number {
|
||||
return this.#blockSize
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
|
||||
const sourceIndexes = this.source.getBlockIndexes()
|
||||
const indexes: Array<number> = []
|
||||
const maxIndex = Math.ceil(this.getVirtualSize() / this.getBlockSize())
|
||||
for (const sourceIndex of sourceIndexes) {
|
||||
for (let i = 0; i < blockRatio; i++) {
|
||||
const index = sourceIndex * blockRatio + i
|
||||
if (index < maxIndex) {
|
||||
indexes.push(index)
|
||||
}
|
||||
}
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
|
||||
hasBlock(index: number): boolean {
|
||||
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
|
||||
const sourceIndex = Math.floor(index / blockRatio)
|
||||
return this.source.hasBlock(sourceIndex)
|
||||
}
|
||||
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
if (this.source instanceof RandomAccessDisk) {
|
||||
const blockRatio = this.source.getBlockSize() / this.getBlockSize()
|
||||
const sourceBlockIndex = Math.floor(index / blockRatio)
|
||||
const indexInblock = index % blockRatio
|
||||
let block = this.#blockCache
|
||||
if (block?.index !== sourceBlockIndex) {
|
||||
this.#blockCache = block = await this.source.readBlock(sourceBlockIndex)
|
||||
}
|
||||
|
||||
const data = block.data.subarray(indexInblock * this.#blockSize, (indexInblock + 1) * this.#blockSize)
|
||||
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
} else {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { test } from 'node:test'
|
||||
import { DiskSmallerBlock } from './DiskSmallerBlock.mjs'
|
||||
import { RandomAccessDisk, DiskBlock, Disk } from './Disk.mjs'
|
||||
|
||||
// Mock implementation of RandomAccessDisk for testing
|
||||
class MockDisk extends RandomAccessDisk {
|
||||
private blockSize: number
|
||||
private virtualSize: number
|
||||
private blocks: Map<number, Buffer>
|
||||
public closed: boolean = false
|
||||
public initialized: boolean = false
|
||||
public isDiff: boolean = false
|
||||
public parentDisk: RandomAccessDisk | null = null
|
||||
|
||||
constructor(blockSize: number, virtualSize: number, blocks: [number, Buffer][] = []) {
|
||||
super()
|
||||
this.blockSize = blockSize
|
||||
this.virtualSize = virtualSize
|
||||
this.blocks = new Map(blocks)
|
||||
}
|
||||
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
if (this.closed) throw new Error('Disk closed')
|
||||
if (!this.initialized) throw new Error('Disk not initialized')
|
||||
if (!this.blocks.has(index)) {
|
||||
if (this.isDiff && this.parentDisk) {
|
||||
return this.parentDisk.readBlock(index)
|
||||
}
|
||||
throw new Error(`Block ${index} not found`)
|
||||
}
|
||||
return { index, data: this.blocks.get(index)! }
|
||||
}
|
||||
|
||||
getVirtualSize(): number {
|
||||
return this.virtualSize
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.blockSize
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
this.initialized = true
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
this.closed = true
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return this.isDiff
|
||||
}
|
||||
instantiateParent(): RandomAccessDisk {
|
||||
if (!this.parentDisk) throw new Error('No parent disk available')
|
||||
return this.parentDisk
|
||||
}
|
||||
getBlockIndexes(): number[] {
|
||||
return [...this.blocks.keys()]
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.blocks.has(index) || (this.isDiff && !!this.parentDisk && this.parentDisk.hasBlock(index))
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create a buffer with pattern
|
||||
function createPatternBuffer(size: number, pattern: string): Buffer {
|
||||
const buf = Buffer.alloc(size)
|
||||
for (let i = 0; i < size; i++) {
|
||||
buf[i] = pattern.charCodeAt(i % pattern.length)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
// Test suite
|
||||
test('constructor and initialization', async () => {
|
||||
const source = new MockDisk(512, 1024 * 1024)
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
|
||||
// Test that blockSize must be a multiple of source blockSize
|
||||
await assert.rejects(
|
||||
async () => {
|
||||
const invalidDisk = new DiskSmallerBlock(source, 452)
|
||||
await invalidDisk.init()
|
||||
},
|
||||
{ message: /must be a multiple/ }
|
||||
)
|
||||
// Test that blockSize must be smaller than source blockSize
|
||||
await assert.rejects(
|
||||
async () => {
|
||||
const invalidDisk = new DiskSmallerBlock(source, 1024 * 2)
|
||||
await invalidDisk.init()
|
||||
},
|
||||
{ message: /smaller/ }
|
||||
)
|
||||
|
||||
await disk.init()
|
||||
assert.strictEqual(disk.getBlockSize(), 256)
|
||||
assert.strictEqual(disk.getVirtualSize(), 1024 * 1024)
|
||||
})
|
||||
|
||||
test('readBlock with simple block mapping', async () => {
|
||||
// Create source disk with two 512-byte blocks
|
||||
const block1 = createPatternBuffer(512, 'A')
|
||||
const block2 = createPatternBuffer(512, 'B')
|
||||
const source = new MockDisk(512, 1024, [
|
||||
[0, block1],
|
||||
[1, block2],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
await disk.init()
|
||||
|
||||
// Read the first block , half
|
||||
const result = await disk.readBlock(2)
|
||||
assert.strictEqual(result.index, 2)
|
||||
assert.strictEqual(result.data.length, 256)
|
||||
|
||||
// Verify the data is correctly combined
|
||||
assert(block2.subarray(0, 256).equals(result.data))
|
||||
})
|
||||
test('readBlock interleaved', async () => {
|
||||
// Create source disk with two 512-byte blocks
|
||||
const block1 = createPatternBuffer(512, 'A')
|
||||
const block2 = createPatternBuffer(512, 'B')
|
||||
const source = new MockDisk(512, 1024, [
|
||||
[0, block1],
|
||||
[1, block2],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
await disk.init()
|
||||
|
||||
// Read the first block , half
|
||||
await disk.readBlock(1)
|
||||
// then second block
|
||||
await disk.readBlock(2)
|
||||
// then back to first
|
||||
const result = await disk.readBlock(0)
|
||||
|
||||
// Verify the data is correctly combined
|
||||
assert(block1.subarray(0, 256).equals(result.data))
|
||||
})
|
||||
|
||||
test('hasBlock behavior', async () => {
|
||||
const source = new MockDisk(512, 2048, [
|
||||
[0, Buffer.alloc(512)],
|
||||
[2, Buffer.alloc(512)],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
await disk.init()
|
||||
|
||||
assert(disk.hasBlock(0))
|
||||
assert(disk.hasBlock(1))
|
||||
assert(!disk.hasBlock(2))
|
||||
assert(!disk.hasBlock(3))
|
||||
assert(disk.hasBlock(4))
|
||||
assert(disk.hasBlock(5))
|
||||
})
|
||||
|
||||
test('getBlockIndexes', async () => {
|
||||
const source = new MockDisk(512, 2048, [
|
||||
[0, Buffer.alloc(512)],
|
||||
[2, Buffer.alloc(512)],
|
||||
])
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
await disk.init()
|
||||
|
||||
const indexes = disk.getBlockIndexes()
|
||||
assert.deepStrictEqual(indexes, [0, 1, 4, 5])
|
||||
})
|
||||
|
||||
test('close propagation', async () => {
|
||||
const source = new MockDisk(512, 1024)
|
||||
await source.init()
|
||||
|
||||
const disk = new DiskSmallerBlock(source, 256)
|
||||
await disk.init()
|
||||
|
||||
await disk.close()
|
||||
assert(source.closed)
|
||||
})
|
||||
@@ -1,26 +0,0 @@
|
||||
import type { Readable } from 'node:stream'
|
||||
|
||||
export type FileDescriptor = number
|
||||
|
||||
/**
|
||||
* a base interface is defined to be able to add other accessor ,
|
||||
* for example in browser or from a vmware datastore
|
||||
*
|
||||
* NOTE : file accessor on datastore should try to use a stream
|
||||
* and read/skip from this when possible, since opening a new stream is costly
|
||||
* and we are generally reading the vhd sequentially
|
||||
*/
|
||||
export interface FileAccessor {
|
||||
closeFile(filehandle: number): Promise<void>
|
||||
openFile(path: string, opts?: unknown): Promise<number>
|
||||
read(
|
||||
file: string | number,
|
||||
buffer: Buffer | DataView,
|
||||
position: number
|
||||
): Promise<{ bytesRead: number; buffer: Buffer | DataView }>
|
||||
getSize(path: string): Promise<number>
|
||||
mktree(path: string): Promise<void>
|
||||
rmtree(path: string): Promise<void>
|
||||
unlink(path: string): Promise<void>
|
||||
outputStream(stream: Readable): Promise<void>
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { DiskBlock, Disk, RandomAccessDisk } from './Disk.mjs'
|
||||
import assert from 'node:assert'
|
||||
|
||||
/**
|
||||
* expose a differencing disk that contains the difference
|
||||
* between child and parent
|
||||
*
|
||||
* the chain parent - child - negative will contains exactly the
|
||||
* same data as parent, this is used to rewind a disk to a previous
|
||||
* state
|
||||
*/
|
||||
export class NegativeDisk extends RandomAccessDisk {
|
||||
#parent: RandomAccessDisk
|
||||
#child: RandomAccessDisk
|
||||
|
||||
constructor(parent: RandomAccessDisk, child: RandomAccessDisk) {
|
||||
super()
|
||||
this.#parent = parent
|
||||
this.#child = child
|
||||
// @todo : should also check chain
|
||||
assert.strictEqual(parent.getBlockSize(), child.getBlockSize(), 'GEOMETRY_CHANGED')
|
||||
assert.strictEqual(parent.getVirtualSize(), child.getVirtualSize(), 'GEOMETRY_CHANGED')
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
return this.#child.getVirtualSize()
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.#child.getBlockSize()
|
||||
}
|
||||
readBlock(index: number): Promise<DiskBlock> {
|
||||
if (this.#parent.hasBlock(index)) {
|
||||
return this.#parent.readBlock(index)
|
||||
}
|
||||
// a new block => return an empty one
|
||||
return Promise.resolve({
|
||||
index,
|
||||
data: Buffer.alloc(this.getBlockSize(), 0),
|
||||
})
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
await Promise.all([this.#parent.init(), this.#child.init()])
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
await Promise.all([this.#parent.close(), this.#child.close()])
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return true
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
return this.#child.getBlockIndexes()
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.#child.hasBlock(index)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export interface ProgressHandler {
|
||||
setProgress(total: number): void | Promise<void>
|
||||
done(): void | Promise<void>
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
import { DiskBlock, RandomAccessDisk } from "./Disk.mjs";
|
||||
import { FileAccessor } from "./FileAccessor.mjs";
|
||||
|
||||
|
||||
export class RawDisk extends RandomAccessDisk{
|
||||
#accessor
|
||||
#path
|
||||
#descriptor:number|undefined
|
||||
#blockSize:number
|
||||
#size: number|undefined
|
||||
|
||||
|
||||
constructor(accessor: FileAccessor, path: string, blockSize:number){
|
||||
super()
|
||||
this.#accessor= accessor
|
||||
this.#path = path
|
||||
this.#blockSize = blockSize
|
||||
}
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
if(this.#descriptor === undefined){
|
||||
throw new Error("Can't call readBlock before init");
|
||||
}
|
||||
const data = Buffer.alloc(this.getBlockSize(), 0)
|
||||
await this.#accessor.read(this.#descriptor, data, index*this.getBlockSize())
|
||||
return {
|
||||
index,
|
||||
data
|
||||
}
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
if(this.#size === undefined){
|
||||
throw new Error("Can't call getVirtualsize before init");
|
||||
}
|
||||
return this.#size
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.#blockSize
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
this.#descriptor = await this.#accessor.openFile(this.#path)
|
||||
this.#size = await this.#accessor.getSize(this.#path)
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
this.#descriptor && await this.#accessor.closeFile(this.#descriptor)
|
||||
this.#descriptor = undefined
|
||||
this.#size = undefined
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return false
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
const nbBlocks = Math.ceil(this.getVirtualSize()/ this.getBlockSize())
|
||||
const index =[]
|
||||
for(let i=0; i < nbBlocks; i ++){
|
||||
index.push(i)
|
||||
}
|
||||
return index
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return index * this.getBlockSize() < this.getVirtualSize()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
import { DiskBlock, RandomAccessDisk } from './Disk.mjs'
|
||||
import { RandomDiskPassthrough } from './DiskPassthrough.mjs'
|
||||
|
||||
export class ReadAhead extends RandomDiskPassthrough {
|
||||
#maxNumber: number
|
||||
#maxPercent: number
|
||||
constructor(source: RandomAccessDisk, { maxNumber = 10, maxPercent = 10 } = {}) {
|
||||
super(source)
|
||||
this.#maxNumber = maxNumber
|
||||
this.#maxPercent = maxPercent
|
||||
}
|
||||
openSource(): Promise<RandomAccessDisk> {
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
buildDiskBlockGenerator(): AsyncGenerator<DiskBlock> {
|
||||
const self = this
|
||||
const blockIndexes = this.getBlockIndexes()
|
||||
const maxNumber = this.#maxNumber
|
||||
const maxPercent = this.#maxPercent
|
||||
|
||||
async function* generator(): AsyncGenerator<DiskBlock> {
|
||||
const preloaded = []
|
||||
// between 1 and 10 or at most 10% of the blocks
|
||||
const PRELOAD_SIZE = Math.max(Math.min(Math.floor((blockIndexes.length * maxPercent) / 100), maxNumber), 1)
|
||||
|
||||
let counter = 0
|
||||
try{
|
||||
for (const index of blockIndexes) {
|
||||
counter++
|
||||
await self.progressHandler?.setProgress(counter/ blockIndexes.length)
|
||||
if (preloaded.length < PRELOAD_SIZE) {
|
||||
preloaded.push(self.source.readBlock(index))
|
||||
}
|
||||
if (preloaded.length === PRELOAD_SIZE) {
|
||||
const next = (await preloaded.shift())!
|
||||
yield next
|
||||
}
|
||||
}
|
||||
while (preloaded.length > 0) {
|
||||
const next = (await preloaded.shift())!
|
||||
yield next
|
||||
}
|
||||
}finally{
|
||||
await self.progressHandler?.done()
|
||||
}
|
||||
|
||||
}
|
||||
return generator()
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import { DiskPassthrough } from './DiskPassthrough.mjs'
|
||||
import { Synchronized } from '@vates/generator-toolbox'
|
||||
import { Disk, DiskBlock } from './Disk.mjs'
|
||||
|
||||
class ForkedDisk extends DiskPassthrough {
|
||||
#generator: AsyncGenerator<DiskBlock, any, any>
|
||||
#generatedDiskBlocks = 0
|
||||
constructor(source: Disk, generator: AsyncGenerator<DiskBlock, any, any>) {
|
||||
super(source)
|
||||
this.#generator = generator
|
||||
}
|
||||
async openSource(): Promise<Disk> {
|
||||
throw new ErrorEvent(' No need to open forked disk ')
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
/* source has already been open , */
|
||||
}
|
||||
async *diskBlocks(): AsyncGenerator<DiskBlock> {
|
||||
try {
|
||||
for await (const block of this.#generator) {
|
||||
this.#generatedDiskBlocks++
|
||||
yield block
|
||||
}
|
||||
} finally {
|
||||
await this.progressHandler?.done()
|
||||
await this.close()
|
||||
}
|
||||
}
|
||||
getNbGeneratedBlock(): number {
|
||||
return this.#generatedDiskBlocks
|
||||
}
|
||||
}
|
||||
|
||||
export class SynchronizedDisk {
|
||||
#synchronized: Synchronized<DiskBlock, any, any> | undefined
|
||||
#source: Disk
|
||||
|
||||
constructor(source: Disk) {
|
||||
this.#source = source
|
||||
}
|
||||
|
||||
fork(uid: string): ForkedDisk {
|
||||
if (this.#synchronized === undefined) {
|
||||
const generator = this.#source.diskBlocks()
|
||||
this.#synchronized = new Synchronized(generator)
|
||||
}
|
||||
return new ForkedDisk(this.#source, this.#synchronized.fork(uid) as AsyncGenerator<DiskBlock, any, any>)
|
||||
}
|
||||
close() {
|
||||
return this.#source.close()
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import { DebugDisk } from './utils/DebugDisk.mjs'
|
||||
import { SynchronizedDisk } from './SynchronizedDisk.mjs'
|
||||
|
||||
async function consumeGenerator(source: AsyncGenerator) {
|
||||
for await (const block of source) {
|
||||
}
|
||||
return 'finished'
|
||||
}
|
||||
test('Disk class', async t => {
|
||||
const testRes = await t.test('synchronised fork disk ', async () => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 20
|
||||
const source = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })
|
||||
|
||||
const disk = new SynchronizedDisk(source)
|
||||
await source.init()
|
||||
assert.strictEqual(source.getVirtualSize(), blockSize * 20, 'size should be source size')
|
||||
assert.strictEqual(source.getBlockSize(), blockSize, 'block size should be source size')
|
||||
const forks = new Array()
|
||||
for (let i = 0; i < 2; i++) {
|
||||
forks.push(disk.fork('generator_' + i))
|
||||
}
|
||||
assert.strictEqual(source.getNbGeneratedBlock(), 0, 'nothing consumed')
|
||||
await Promise.all(forks.map(fork => consumeGenerator(fork.diskBlocks())))
|
||||
await disk.close()
|
||||
assert.strictEqual(source.closeDone, true)
|
||||
assert.strictEqual(source.getNbGeneratedBlock(), nbBlocks)
|
||||
})
|
||||
console.log({ testRes })
|
||||
})
|
||||
@@ -1,11 +0,0 @@
|
||||
export { Disk, DiskBlock, RandomAccessDisk } from './Disk.mjs'
|
||||
export { DiskChain } from './DiskChain.mjs'
|
||||
export { DiskLargerBlock } from './DiskLargerBlock.mjs'
|
||||
export { DiskSmallerBlock } from './DiskSmallerBlock.mjs'
|
||||
export { NegativeDisk } from './NegativeDisk.mjs'
|
||||
export { FileAccessor } from './FileAccessor.mjs'
|
||||
export { DiskPassthrough, RandomDiskPassthrough } from './DiskPassthrough.mjs'
|
||||
export { RawDisk } from './RawDisk.mjs'
|
||||
export { ReadAhead } from './ReadAhead.mjs'
|
||||
export { SynchronizedDisk } from './SynchronizedDisk.mjs'
|
||||
export { ProgressHandler } from './ProgressHandler.mjs'
|
||||
@@ -1,33 +0,0 @@
|
||||
// @ts-expect-error async-each is not typed
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { Disk, DiskBlock } from '../Disk.mjs'
|
||||
import { DebugDisk } from './DebugDisk.mjs'
|
||||
|
||||
/**
|
||||
*
|
||||
* @param disk utility method that consume a disk and show the speed
|
||||
*/
|
||||
|
||||
export async function consume(disk: Disk, { blockDelay = 20, concurrency = 8 } = {}) {
|
||||
const start = Date.now()
|
||||
let consumed = 0,
|
||||
nbBlocks = 0
|
||||
|
||||
await asyncEach(
|
||||
disk.diskBlocks(),
|
||||
async (block: DiskBlock) => {
|
||||
consumed += block.data.length
|
||||
nbBlocks++
|
||||
await new Promise(resolve => setTimeout(resolve, blockDelay))
|
||||
},
|
||||
{ concurrency }
|
||||
)
|
||||
const end = Date.now()
|
||||
const duration = end - start
|
||||
console.log('consume', {
|
||||
duration, // in ms
|
||||
nbBlocks,
|
||||
consumed: Math.round(consumed / 1024 / 1024), // in MB
|
||||
speed: Math.round((consumed * 1000) / duration / 1024 / 1024), // in MB/s
|
||||
})
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
import { Disk, DiskBlock, RandomAccessDisk } from '../Disk.mjs'
|
||||
|
||||
/**
|
||||
* utility class that generate a random disk
|
||||
*/
|
||||
export class DebugDisk extends RandomAccessDisk {
|
||||
#nbBlocks: number
|
||||
#blockSize: number
|
||||
#fillRate: number // 0 - 100
|
||||
#filledBy: number // 0 255
|
||||
blocks: Array<boolean> = []
|
||||
closeDone = false
|
||||
initDone = false
|
||||
constructor({
|
||||
nbBlocks,
|
||||
blockSize,
|
||||
fillRate,
|
||||
filledBy = 0,
|
||||
}: {
|
||||
nbBlocks: number
|
||||
blockSize: number
|
||||
fillRate: number
|
||||
filledBy?: number
|
||||
}) {
|
||||
super()
|
||||
this.#nbBlocks = nbBlocks
|
||||
this.#blockSize = blockSize
|
||||
this.#fillRate = fillRate
|
||||
this.#filledBy = filledBy
|
||||
}
|
||||
getVirtualSize(): number {
|
||||
return this.#blockSize * this.#nbBlocks
|
||||
}
|
||||
getBlockSize(): number {
|
||||
return this.#blockSize
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
for (let i = 0; i < this.#nbBlocks; i++) {
|
||||
this.blocks.push(Math.random() <= this.#fillRate / 100)
|
||||
}
|
||||
this.initDone = true
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
this.closeDone = true
|
||||
}
|
||||
isDifferencing(): boolean {
|
||||
return false
|
||||
}
|
||||
getBlockIndexes(): Array<number> {
|
||||
const indexes = []
|
||||
for (let i = 0; i < this.#nbBlocks; i++) {
|
||||
if (this.blocks[i]) {
|
||||
indexes.push(i)
|
||||
}
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
hasBlock(index: number): boolean {
|
||||
return this.blocks[index]
|
||||
}
|
||||
async readBlock(index: number): Promise<DiskBlock> {
|
||||
return {
|
||||
index,
|
||||
data: Buffer.alloc(this.getBlockSize(), this.#filledBy),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
|
||||
"strict": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "es2022",
|
||||
"skipLibCheck": true,
|
||||
"declaration": true // Generates `.d.ts` files for type safety
|
||||
},
|
||||
"include": ["./src/**/*"],
|
||||
"exclude": ["./node_modules", "./dist"]
|
||||
}
|
||||
@@ -14,7 +14,7 @@ ee.on('start', async function () {
|
||||
// whatever
|
||||
})
|
||||
|
||||
// similar to EventEmitter#emit() but returns a promise which resolves when all
|
||||
// similar to EventEmmiter#emit() but returns a promise which resolves when all
|
||||
// listeners have settled
|
||||
await ee.emitAsync('start')
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ ee.on('start', async function () {
|
||||
// whatever
|
||||
})
|
||||
|
||||
// similar to EventEmitter#emit() but returns a promise which resolves when all
|
||||
// similar to EventEmmiter#emit() but returns a promise which resolves when all
|
||||
// listeners have settled
|
||||
await ee.emitAsync('start')
|
||||
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
## Testing Azure Blob Storage Using Azurite emulator
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- A Linux-based VM with Node.js installed
|
||||
|
||||
### Steps
|
||||
|
||||
1. Install Azurite
|
||||
|
||||
```sh
|
||||
npm install -g azurite
|
||||
```
|
||||
|
||||
2. Run Azurite
|
||||
|
||||
```sh
|
||||
azurite --blobHost 0.0.0.0 --silent --location c:\azurite --debug c:\azurite\debug.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**Explanation of Flags:**
|
||||
|
||||
- `--blobHost 0.0.0.0` → Binds Azurite to all network interfaces making it accessible from external machines
|
||||
- `--silent` → Suppresses unnecessary logs
|
||||
- `--location c:\azurite` → Specifies the directory where Azurite will store data
|
||||
- `--debug c:\azurite\debug.log` → Enables debug logging for troubleshooting
|
||||
|
||||
---
|
||||
|
||||
3. Configure Connection String on Your Local Machine
|
||||
|
||||
Replace `AZURITE_CONNECTION_STRING` with the following connection string in the storage operations commands:
|
||||
|
||||
`DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://<VM_IP>:10000/devstoreaccount1;QueueEndpoint=http://<VM_IP>:10001/devstoreaccount1;TableEndpoint=http://<VM_IP>:10002/devstoreaccount1;`
|
||||
|
||||
**Replace `<VM_IP>` with the actual IP address of your VM where Azurite is running.**
|
||||
|
||||
**The account name (devstoreaccount1) and key provided in the connection string above are the default credentials for Azurite. These credentials are publicly documented and can be used by anyone for local development and testing**
|
||||
|
||||
---
|
||||
|
||||
### 4. Perform Storage Operations
|
||||
|
||||
Create a Container (Equivalent to an S3 Bucket)
|
||||
|
||||
```sh
|
||||
az storage container create --name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
Upload a Blob (Equivalent to a File/Directory in S3)
|
||||
|
||||
```sh
|
||||
az storage blob upload --account-name devstoreaccount1 --container-name <container_name> --name <blob_name> --file <file_path> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
List All Containers
|
||||
|
||||
```sh
|
||||
az storage container list --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
Download a Blob
|
||||
|
||||
```sh
|
||||
az storage blob download --container-name <container_name> --name <blob_name> --file <file_name> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
Delete a Blob
|
||||
|
||||
```sh
|
||||
az storage blob delete --container-name <container_name> --name <blob_name> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
List Blobs Inside a Container
|
||||
|
||||
```sh
|
||||
az storage blob list --container-name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
Delete a Container
|
||||
|
||||
```sh
|
||||
az storage container delete --name <container_name> --connection-string "AZURITE_CONNECTION_STRING"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Notes
|
||||
|
||||
- Ensure your VM allows incoming connections on ports `10000`, `10001`, and `10002` for the Blob, Queue, and Table endpoints, respectively
|
||||
- Use `--debug` flag while running commands if you need detailed logs
|
||||
- Replace placeholders (`<VM_IP>`, `<container_name>`, `<blob_name>`, `<file_path>`, `<file_name>`) with actual values before executing commands
|
||||
|
||||
Source documentation: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=npm%2Cblob-storage
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/fs",
|
||||
"version": "4.6.2",
|
||||
"version": "4.5.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "The File System for Xen Orchestra backups.",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
|
||||
@@ -25,7 +25,6 @@
|
||||
"@aws-sdk/lib-storage": "^3.54.0",
|
||||
"@aws-sdk/middleware-apply-body-checksum": "^3.58.0",
|
||||
"@aws-sdk/node-http-handler": "^3.54.0",
|
||||
"@azure/storage-blob": "^12.26.0",
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@vates/coalesce-calls": "^0.1.0",
|
||||
"@vates/decorate-with": "^2.1.0",
|
||||
@@ -42,7 +41,7 @@
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"readable-stream": "^4.1.0",
|
||||
"through2": "^4.0.2",
|
||||
"xo-remote-parser": "^0.10.0"
|
||||
"xo-remote-parser": "^0.9.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.0.0",
|
||||
|
||||
@@ -3,14 +3,13 @@ import getStream from 'get-stream'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { coalesceCalls } from '@vates/coalesce-calls'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { fromCallback, fromEvent, ignoreErrors, pRetry } from 'promise-toolbox'
|
||||
import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { parse } from 'xo-remote-parser'
|
||||
import { pipeline } from 'stream'
|
||||
import { randomBytes, randomUUID } from 'crypto'
|
||||
import { synchronized } from 'decorator-synchronized'
|
||||
|
||||
import { withTimeout } from './utils'
|
||||
import { basename, dirname, normalize as normalizePath } from './path'
|
||||
import { createChecksumStream, validChecksumOfReadStream } from './checksum'
|
||||
import { DEFAULT_ENCRYPTION_ALGORITHM, UNENCRYPTED_ALGORITHM, _getEncryptor } from './_encryptor'
|
||||
@@ -29,58 +28,6 @@ const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
|
||||
const ENCRYPTION_DESC_FILENAME = 'encryption.json'
|
||||
const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
|
||||
|
||||
const WITH_LIMIT = [
|
||||
'closeFile',
|
||||
'copy',
|
||||
'getInfo',
|
||||
'getSizeOnDisk',
|
||||
'list',
|
||||
'mkdir',
|
||||
'openFile',
|
||||
'outputFile',
|
||||
'read',
|
||||
'readFile',
|
||||
'rename',
|
||||
'rmdir',
|
||||
'truncate',
|
||||
'unlink',
|
||||
'write',
|
||||
'writeFile',
|
||||
]
|
||||
|
||||
const WITH_RETRY = [
|
||||
'_closeFile',
|
||||
'_copy',
|
||||
'_getInfo',
|
||||
'_getSize',
|
||||
'_list',
|
||||
'_mkdir',
|
||||
'_openFile',
|
||||
'_outputFile',
|
||||
'_read',
|
||||
'_readFile',
|
||||
'_rename',
|
||||
'_rmdir',
|
||||
'_truncate',
|
||||
'_unlink',
|
||||
'_write',
|
||||
'_writeFile',
|
||||
]
|
||||
|
||||
const WITH_TIMEOUT = [
|
||||
'_closeFile',
|
||||
'_createReadStream',
|
||||
'_getInfo',
|
||||
'_getSize',
|
||||
'_list',
|
||||
'_rename',
|
||||
'_copy',
|
||||
'_rmdir',
|
||||
'_closeFile',
|
||||
'_openFile',
|
||||
'_createOutputStream',
|
||||
]
|
||||
|
||||
const ignoreEnoent = error => {
|
||||
if (error == null || error.code !== 'ENOENT') {
|
||||
throw error
|
||||
@@ -146,59 +93,28 @@ export default class RemoteHandlerAbstract {
|
||||
throw new Error('Incorrect remote type')
|
||||
}
|
||||
}
|
||||
;({
|
||||
highWaterMark: this._highWaterMark,
|
||||
timeout: this._timeout = DEFAULT_TIMEOUT,
|
||||
withLimit: this._withLimit = WITH_LIMIT,
|
||||
withTimeout: this._withTimeout = WITH_TIMEOUT,
|
||||
withRetry: this._withRetry = WITH_RETRY,
|
||||
} = options)
|
||||
;({ highWaterMark: this._highWaterMark, timeout: this._timeout = DEFAULT_TIMEOUT } = options)
|
||||
|
||||
this.#applySafeGuards(options)
|
||||
}
|
||||
|
||||
#conditionRetry(error) {
|
||||
return !['EEXIST', 'EISDIR', 'ENOTEMPTY', 'ENOENT', 'ENOTDIR', 'SystemInUse', 'ERR_ASSERTION'].includes(error?.code)
|
||||
}
|
||||
|
||||
#applySafeGuards(options) {
|
||||
const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
|
||||
this.closeFile = sharedLimit(this.closeFile)
|
||||
this.copy = sharedLimit(this.copy)
|
||||
this.getInfo = sharedLimit(this.getInfo)
|
||||
this.getSizeOnDisk = sharedLimit(this.getSizeOnDisk)
|
||||
this.list = sharedLimit(this.list)
|
||||
this.mkdir = sharedLimit(this.mkdir)
|
||||
this.openFile = sharedLimit(this.openFile)
|
||||
this.outputFile = sharedLimit(this.outputFile)
|
||||
this.read = sharedLimit(this.read)
|
||||
this.readFile = sharedLimit(this.readFile)
|
||||
this.rename = sharedLimit(this.rename)
|
||||
this.rmdir = sharedLimit(this.rmdir)
|
||||
this.truncate = sharedLimit(this.truncate)
|
||||
this.unlink = sharedLimit(this.unlink)
|
||||
this.write = sharedLimit(this.write)
|
||||
this.writeFile = sharedLimit(this.writeFile)
|
||||
|
||||
this._forget = coalesceCalls(this._forget)
|
||||
this._sync = coalesceCalls(this._sync)
|
||||
|
||||
this._withLimit.forEach(functionName => {
|
||||
if (this[functionName] !== undefined) {
|
||||
this[functionName] = sharedLimit(this[functionName])
|
||||
}
|
||||
})
|
||||
|
||||
this._withTimeout.forEach(functionName => {
|
||||
if (this[functionName] !== undefined) {
|
||||
this[functionName] = withTimeout(this[functionName], DEFAULT_TIMEOUT)
|
||||
}
|
||||
})
|
||||
|
||||
this._withRetry.forEach(functionName => {
|
||||
if (this[functionName] !== undefined) {
|
||||
// adding the retry on the top level method won't
|
||||
// cover when _functionName are called internally
|
||||
this[functionName] = pRetry.wrap(this[functionName], {
|
||||
delays: [100, 200, 500, 1000, 2000],
|
||||
// these errors should not change on retry
|
||||
when: err => this.#conditionRetry(err),
|
||||
onRetry(error) {
|
||||
warn('retrying method on fs ', {
|
||||
method: functionName,
|
||||
attemptNumber: this.attemptNumber,
|
||||
delay: this.delay,
|
||||
error,
|
||||
file: this.arguments?.[0],
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Public members
|
||||
@@ -224,7 +140,11 @@ export default class RemoteHandlerAbstract {
|
||||
file = normalizePath(file)
|
||||
}
|
||||
|
||||
let stream = await this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark })
|
||||
let stream = await timeout.call(
|
||||
this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }),
|
||||
this._timeout
|
||||
)
|
||||
|
||||
// detect early errors
|
||||
await fromEvent(stream, 'readable')
|
||||
|
||||
@@ -245,18 +165,14 @@ export default class RemoteHandlerAbstract {
|
||||
|
||||
if (this.isEncrypted) {
|
||||
stream = this.#encryptor.decryptStream(stream)
|
||||
}
|
||||
|
||||
// try to add the length prop if missing and not a range stream
|
||||
if (stream.length === undefined && options.end === undefined && options.start === undefined) {
|
||||
try {
|
||||
if (this.isEncrypted) {
|
||||
stream.maxStreamLength = await this.getSizeOnDisk(file)
|
||||
} else {
|
||||
} else {
|
||||
// try to add the length prop if missing and not a range stream
|
||||
if (stream.length === undefined && options.end === undefined && options.start === undefined) {
|
||||
try {
|
||||
stream.length = await this._getSize(file)
|
||||
} catch (error) {
|
||||
// ignore errors
|
||||
}
|
||||
} catch (error) {
|
||||
// ignore errors
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,16 +206,16 @@ export default class RemoteHandlerAbstract {
|
||||
validator,
|
||||
})
|
||||
if (checksum) {
|
||||
// using _outputFile means the checksum will NOT be encrypted
|
||||
// using _outpuFile means the checksum will NOT be encrypted
|
||||
// it is by design to allow checking of encrypted files without the key
|
||||
await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' })
|
||||
}
|
||||
}
|
||||
|
||||
// Free the resources possibly dedicated to put the remote at work, when it
|
||||
// is no longer needed
|
||||
// is no more needed
|
||||
//
|
||||
// FIXME: Some handlers are implemented based on system-wide mechanisms (such
|
||||
// FIXME: Some handlers are implemented based on system-wide mecanisms (such
|
||||
// as mount), forgetting them might breaking other processes using the same
|
||||
// remote.
|
||||
@synchronized()
|
||||
@@ -308,7 +224,7 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async getInfo() {
|
||||
return this._getInfo()
|
||||
return timeout.call(this._getInfo(), this._timeout)
|
||||
}
|
||||
|
||||
// returns the real size occupied by an unencrypted file
|
||||
@@ -319,7 +235,7 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async getSizeOnDisk(file) {
|
||||
return this._getSize(typeof file === 'string' ? normalizePath(file) : file)
|
||||
return timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
|
||||
}
|
||||
|
||||
async __list(dir, { filter, ignoreMissing = false, prependDir = false } = {}) {
|
||||
@@ -327,7 +243,7 @@ export default class RemoteHandlerAbstract {
|
||||
const virtualDir = normalizePath(dir)
|
||||
dir = normalizePath(dir)
|
||||
|
||||
let entries = await this._list(dir)
|
||||
let entries = await timeout.call(this._list(dir), this._timeout)
|
||||
if (filter !== undefined) {
|
||||
entries = entries.filter(filter)
|
||||
}
|
||||
@@ -373,7 +289,7 @@ export default class RemoteHandlerAbstract {
|
||||
|
||||
async #rename(oldPath, newPath, { checksum }, createTree = true) {
|
||||
try {
|
||||
let p = this._rename(oldPath, newPath)
|
||||
let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
|
||||
if (checksum) {
|
||||
p = Promise.all([p, this._rename(checksumFile(oldPath), checksumFile(newPath))])
|
||||
}
|
||||
@@ -396,7 +312,7 @@ export default class RemoteHandlerAbstract {
|
||||
oldPath = normalizePath(oldPath)
|
||||
newPath = normalizePath(newPath)
|
||||
|
||||
let p = this._copy(oldPath, newPath)
|
||||
let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
|
||||
if (checksum) {
|
||||
p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
|
||||
}
|
||||
@@ -404,14 +320,14 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async rmdir(dir) {
|
||||
await this._rmdir(normalizePath(dir)).catch(ignoreEnoent)
|
||||
await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
|
||||
}
|
||||
|
||||
async rmtree(dir) {
|
||||
await this._rmtree(normalizePath(dir))
|
||||
}
|
||||
|
||||
// Asks the handler to sync the state of the effective remote with its
|
||||
// Asks the handler to sync the state of the effective remote with its'
|
||||
// metadata
|
||||
//
|
||||
// This method MUST ALWAYS be called before using the handler.
|
||||
@@ -466,7 +382,7 @@ export default class RemoteHandlerAbstract {
|
||||
const data = await this.__readFile(ENCRYPTION_METADATA_FILENAME)
|
||||
JSON.parse(data)
|
||||
} catch (error) {
|
||||
// can be enoent, bad algorithm, or broken json ( bad key or algorithm)
|
||||
// can be enoent, bad algorithm, or broeken json ( bad key or algorithm)
|
||||
if (encryptionAlgorithm !== 'none') {
|
||||
if (await this.#canWriteMetadata()) {
|
||||
// any other error , but on empty remote => update with remote settings
|
||||
@@ -474,7 +390,7 @@ export default class RemoteHandlerAbstract {
|
||||
info('will update metadata of this remote')
|
||||
return this.#createMetadata()
|
||||
} else {
|
||||
// to add a new encrypted fs remote, the remote directory must be empty; otherwise, metadata.json is not created
|
||||
// to add a new encrypted fs remote, the remote directory must be empty, otherwise metadata.json is not created
|
||||
if (error.code === 'ENOENT' && error.path.includes('metadata.json')) {
|
||||
throw new Error('Remote directory must be empty.')
|
||||
}
|
||||
@@ -553,7 +469,7 @@ export default class RemoteHandlerAbstract {
|
||||
// Methods that can be called by private methods to avoid parallel limit on public methods
|
||||
|
||||
async __closeFile(fd) {
|
||||
await this._closeFile(fd.fd)
|
||||
await timeout.call(this._closeFile(fd.fd), this._timeout)
|
||||
}
|
||||
|
||||
async __mkdir(dir, { mode } = {}) {
|
||||
@@ -575,7 +491,7 @@ export default class RemoteHandlerAbstract {
|
||||
path = normalizePath(path)
|
||||
|
||||
return {
|
||||
fd: await this._openFile(path, flags),
|
||||
fd: await timeout.call(this._openFile(path, flags), this._timeout),
|
||||
path,
|
||||
}
|
||||
}
|
||||
@@ -667,11 +583,14 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async _outputStream(path, input, { dirMode, validator }) {
|
||||
const tmpPath = `${dirname(path)}/.${Date.now()}.${basename(path)}`
|
||||
const output = await this._createOutputStream(tmpPath, {
|
||||
dirMode,
|
||||
flags: 'wx',
|
||||
})
|
||||
const tmpPath = `${dirname(path)}/.${basename(path)}`
|
||||
const output = await timeout.call(
|
||||
this._createOutputStream(tmpPath, {
|
||||
dirMode,
|
||||
flags: 'wx',
|
||||
}),
|
||||
this._timeout
|
||||
)
|
||||
try {
|
||||
await fromCallback(pipeline, input, output)
|
||||
if (validator !== undefined) {
|
||||
@@ -713,25 +632,25 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
const files = await this._list(dir)
|
||||
await asyncEach(
|
||||
files,
|
||||
file =>
|
||||
this._unlink(`${dir}/${file}`).catch(error => {
|
||||
await asyncEach(files, file =>
|
||||
this._unlink(`${dir}/${file}`).catch(
|
||||
error => {
|
||||
// Unlink dir behavior is not consistent across platforms
|
||||
// https://github.com/nodejs/node-v0.x-archive/issues/5791
|
||||
if (error.code === 'EISDIR' || error.code === 'EPERM') {
|
||||
return this._rmtree(`${dir}/${file}`).catch(rmTreeError => {
|
||||
if (rmTreeError.code === 'ENOTDIR') {
|
||||
// this was really a EPERM error, maybe with immutable backups
|
||||
// this was realy a EPERM error, maybe with immutable backups
|
||||
throw error
|
||||
}
|
||||
throw rmTreeError
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}),
|
||||
// real unlink concurrency will be 2**max directory depth
|
||||
{ concurrency: 2 }
|
||||
},
|
||||
// real unlink concurrency will be 2**max directory depth
|
||||
{ concurrency: 2 }
|
||||
)
|
||||
)
|
||||
return this._rmtree(dir)
|
||||
}
|
||||
|
||||
@@ -10,37 +10,17 @@ import AbstractHandler from './abstract'
|
||||
import fs from 'fs-extra'
|
||||
import tmp from 'tmp'
|
||||
|
||||
const TIMEOUT = 6e5
|
||||
const TIMEOUT = 10e3
|
||||
|
||||
class TestHandler extends AbstractHandler {
|
||||
constructor() {
|
||||
const options = { timeout: TIMEOUT, withRetry: [] }
|
||||
super({ url: 'test://' }, options)
|
||||
constructor(impl) {
|
||||
super({ url: 'test://' }, { timeout: TIMEOUT })
|
||||
Object.defineProperty(this, 'isEncrypted', {
|
||||
get: () => false, // encryption is tested separately
|
||||
get: () => false, // encryption is tested separatly
|
||||
})
|
||||
Object.keys(impl).forEach(method => {
|
||||
this[`_${method}`] = impl[method]
|
||||
})
|
||||
}
|
||||
|
||||
async _closeFile() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _getInfo() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _getSize() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _list() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _openFile() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _rename() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _rmdir() {
|
||||
Promise(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +30,9 @@ const clock = sinon.useFakeTimers()
|
||||
|
||||
describe('closeFile()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
closeFile: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.closeFile({ fd: undefined, path: '' })
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -60,7 +42,9 @@ describe('closeFile()', () => {
|
||||
|
||||
describe('getInfo()', () => {
|
||||
it('throws in case of timeout', async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
getInfo: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.getInfo()
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -70,7 +54,9 @@ describe('getInfo()', () => {
|
||||
|
||||
describe('getSize()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
getSize: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.getSize('')
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -80,7 +66,9 @@ describe('getSize()', () => {
|
||||
|
||||
describe('list()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
list: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.list('.')
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -90,7 +78,9 @@ describe('list()', () => {
|
||||
|
||||
describe('openFile()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
openFile: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.openFile('path')
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -100,7 +90,9 @@ describe('openFile()', () => {
|
||||
|
||||
describe('rename()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
rename: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.rename('oldPath', 'newPath')
|
||||
clock.tick(TIMEOUT)
|
||||
@@ -110,7 +102,9 @@ describe('rename()', () => {
|
||||
|
||||
describe('rmdir()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
const testHandler = new TestHandler({
|
||||
rmdir: () => new Promise(() => {}),
|
||||
})
|
||||
|
||||
const promise = testHandler.rmdir('dir')
|
||||
clock.tick(TIMEOUT)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user