Compare commits

..

1 Commits

Author SHA1 Message Date
Florent BEAUCHAMP
f51e25ac28 fix(XenApi): cannot read properties of undefined (reading 'statusCode') 2023-08-29 08:48:40 +02:00
1058 changed files with 19457 additions and 50434 deletions

View File

@@ -15,10 +15,9 @@ module.exports = {
overrides: [
{
files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js', '**/scripts/**.{,c,m}js'],
files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
rules: {
'n/no-process-exit': 'off',
'n/shebang': 'off',
'no-console': 'off',
},
},
@@ -47,58 +46,6 @@ module.exports = {
],
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/**/*.{vue,ts}'],
parserOptions: {
sourceType: 'module',
},
plugins: ['import'],
extends: [
'plugin:import/recommended',
'plugin:import/typescript',
'plugin:vue/vue3-recommended',
'@vue/eslint-config-typescript/recommended',
'@vue/eslint-config-prettier',
],
settings: {
'import/resolver': {
typescript: true,
'eslint-import-resolver-custom-alias': {
alias: {
'@core': '../web-core/lib',
'@': './src',
},
extensions: ['.ts'],
packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
},
},
},
rules: {
'no-void': 'off',
'n/no-missing-import': 'off', // using 'import' plugin instead to support TS aliases
'@typescript-eslint/no-explicit-any': 'off',
'vue/require-default-prop': 'off', // https://github.com/vuejs/eslint-plugin-vue/issues/2051
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/src/pages/**/*.vue'],
parserOptions: {
sourceType: 'module',
},
rules: {
'vue/multi-word-component-names': 'off',
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
parserOptions: {
sourceType: 'module',
},
rules: {
'eslint-comments/disable-enable-pair': 'off',
'eslint-comments/no-unlimited-disable': 'off',
},
},
],
parserOptions: {
@@ -121,11 +68,6 @@ module.exports = {
'no-console': ['error', { allow: ['warn', 'error'] }],
// this rule can prevent race condition bugs like parallel `a += await foo()`
//
// as it has a lots of false positive, it is only enabled as a warning for now
'require-atomic-updates': 'warn',
strict: 'error',
},
}

48
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,48 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
assignees: ''
---
1. ⚠️ **If you don't follow this template, the issue will be closed**.
2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
Are you using XOA or XO from the sources?
If XOA:
- which release channel? (`stable` vs `latest`)
- please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
If XO from the sources:
- Provide **your commit number**. If it's older than a week, we won't investigate
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Environment (please provide the following information):**
- Node: [e.g. 16.12.1]
- hypervisor: [e.g. XCP-ng 8.2.0]
**Additional context**
Add any other context about the problem here.

View File

@@ -1,119 +0,0 @@
name: Bug Report
description: Create a report to help us improve
labels: ['type: bug :bug:', 'status: triaging :triangular_flag_on_post:']
body:
- type: markdown
attributes:
value: |
1. ⚠️ **If you don't follow this template, the issue will be closed**.
2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
- type: markdown
attributes:
value: '## Are you using XOA or XO from the sources?'
- type: dropdown
id: xo-origin
attributes:
label: Are you using XOA or XO from the sources?
options:
- XOA
- XO from the sources
- both
validations:
required: false
- type: markdown
attributes:
value: '### If XOA:'
- type: dropdown
id: xoa-channel
attributes:
label: Which release channel?
description: please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
options:
- stable
- latest
- both
validations:
required: false
- type: markdown
attributes:
value: '### If XO from the sources:'
- type: markdown
attributes:
value: |
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
- type: input
id: xo-sources-commit-number
attributes:
label: Provide your commit number
description: If it's older than a week, we won't investigate
placeholder: e.g. 579f0
validations:
required: false
- type: markdown
attributes:
value: '## Bug description:'
- type: textarea
id: bug-description
attributes:
label: Describe the bug
description: A clear and concise description of what the bug is
validations:
required: true
- type: textarea
id: error-message
attributes:
label: Error message
render: Text
validations:
required: false
- type: textarea
id: steps
attributes:
label: To reproduce
description: 'Steps to reproduce the behavior:'
value: |
1. Go to '...'
2. Click on '...'
3. Scroll down to '...'
4. See error
validations:
required: false
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
description: A clear and concise description of what you expected to happen
validations:
required: false
- type: textarea
id: screenshots
attributes:
label: Screenshots
description: If applicable, add screenshots to help explain your problem
validations:
required: false
- type: markdown
attributes:
value: '## Environment (please provide the following information):'
- type: input
id: node-version
attributes:
label: Node
placeholder: e.g. 16.12.1
validations:
required: true
- type: input
id: hypervisor-version
attributes:
label: Hypervisor
placeholder: e.g. XCP-ng 8.2.0
validations:
required: true
- type: textarea
id: additional-context
attributes:
label: Additional context
description: Add any other context about the problem here
validations:
required: false

View File

@@ -24,14 +24,8 @@ jobs:
cache: 'yarn'
- name: Install project dependencies
run: yarn
- name: Ensure packages are sorted in CHANGELOG.unreleased.md
run: ./scripts/gen-deps-list.js --check-order > /dev/null
- name: Ensure yarn.lock is up-to-date
run: git diff --exit-code yarn.lock
- name: Build the project
run: yarn build
- name: Unit tests
run: yarn test-unit
- name: Lint tests
run: yarn test-lint
- name: Integration tests

4
.gitignore vendored
View File

@@ -30,12 +30,8 @@ pnpm-debug.log.*
yarn-error.log
yarn-error.log.*
.env
*.tsbuildinfo
# code coverage
.nyc_output/
coverage/
.turbo/
# https://node-tap.org/dot-tap-folder/
.tap/

View File

@@ -33,7 +33,8 @@
"test": "node--test"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^15.0.1",
"tap": "^16.3.0",
"test": "^3.2.1"
}
}

View File

@@ -62,42 +62,6 @@ decorateClass(Foo, {
})
```
### `decorateObject(object, map)`
Decorates an object the same way `decorateClass()` decorates a class:
```js
import { decorateObject } from '@vates/decorate-with'
const object = {
get bar() {
// body
},
set bar(value) {
// body
},
baz() {
// body
},
}
decorateObject(object, {
// getter and/or setter
bar: {
// without arguments
get: lodash.memoize,
// with arguments
set: [lodash.debounce, 150],
},
// method (with or without arguments)
baz: lodash.curry,
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.

View File

@@ -80,42 +80,6 @@ decorateClass(Foo, {
})
```
### `decorateObject(object, map)`
Decorates an object the same way `decorateClass()` decorates a class:
```js
import { decorateObject } from '@vates/decorate-with'
const object = {
get bar() {
// body
},
set bar(value) {
// body
},
baz() {
// body
},
}
decorateObject(object, {
// getter and/or setter
bar: {
// without arguments
get: lodash.memoize,
// with arguments
set: [lodash.debounce, 150],
},
// method (with or without arguments)
baz: lodash.curry,
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.

View File

@@ -14,13 +14,10 @@ function applyDecorator(decorator, value) {
}
exports.decorateClass = exports.decorateMethodsWith = function decorateClass(klass, map) {
return decorateObject(klass.prototype, map)
}
function decorateObject(object, map) {
const { prototype } = klass
for (const name of Object.keys(map)) {
const decorator = map[name]
const descriptor = getOwnPropertyDescriptor(object, name)
const descriptor = getOwnPropertyDescriptor(prototype, name)
if (typeof decorator === 'function' || Array.isArray(decorator)) {
descriptor.value = applyDecorator(decorator, descriptor.value)
} else {
@@ -33,11 +30,10 @@ function decorateObject(object, map) {
}
}
defineProperty(object, name, descriptor)
defineProperty(prototype, name, descriptor)
}
return object
return klass
}
exports.decorateObject = decorateObject
exports.perInstance = function perInstance(fn, decorator, ...args) {
const map = new WeakMap()

View File

@@ -13,15 +13,12 @@ describe('decorateWith', () => {
const expectedFn = Function.prototype
const newFn = () => {}
const decorator = decorateWith(
function wrapper(fn, ...args) {
assert.deepStrictEqual(fn, expectedFn)
assert.deepStrictEqual(args, expectedArgs)
const decorator = decorateWith(function wrapper(fn, ...args) {
assert.deepStrictEqual(fn, expectedFn)
assert.deepStrictEqual(args, expectedArgs)
return newFn
},
...expectedArgs
)
return newFn
}, ...expectedArgs)
const descriptor = {
configurable: true,

View File

@@ -20,7 +20,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "2.1.0",
"version": "2.0.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -14,7 +14,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.1.5",
"version": "0.1.4",
"engines": {
"node": ">=8.10"
},
@@ -23,13 +23,13 @@
"test": "node--test"
},
"dependencies": {
"@vates/multi-key-map": "^0.2.0",
"@vates/multi-key-map": "^0.1.0",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.6.0",
"ensure-array": "^1.0.0"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^15.0.1",
"test": "^3.2.1"
}
}

View File

@@ -20,9 +20,6 @@ function assertListeners(t, event, listeners) {
}
t.beforeEach(function (t) {
// work around https://github.com/tapjs/tapjs/issues/998
t.context = {}
t.context.ee = new EventEmitter()
t.context.em = new EventListenersManager(t.context.ee)
})

View File

@@ -38,9 +38,9 @@
"version": "1.0.1",
"scripts": {
"postversion": "npm publish --access public",
"test": "tap --allow-incomplete-coverage"
"test": "tap --branches=72"
},
"devDependencies": {
"tap": "^18.7.0"
"tap": "^16.2.0"
}
}

View File

@@ -1,28 +0,0 @@
Mount a vhd generated by xen-orchestra to filesystem
### Library
```js
import { mount } from 'fuse-vhd'
// return a disposable, see promise-toolbox/Disposable
// unmount automatically when disposable is disposed
// in case of differencing VHD, it mounts the full chain
await mount(handler, diskId, mountPoint)
```
### cli
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
```
After installing the package
```
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`

View File

@@ -1,59 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/fuse-vhd
[![Package Version](https://badgen.net/npm/v/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd) ![License](https://badgen.net/npm/license/@vates/fuse-vhd) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/fuse-vhd)](https://bundlephobia.com/result?p=@vates/fuse-vhd) [![Node compatibility](https://badgen.net/npm/node/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd)
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/fuse-vhd):
```sh
npm install --save @vates/fuse-vhd
```
## Usage
Mount a vhd generated by xen-orchestra to filesystem
### Library
```js
import { mount } from 'fuse-vhd'
// return a disposable, see promise-toolbox/Disposable
// unmount automatically when disposable is disposed
// in case of differencing VHD, it mounts the full chain
await mount(handler, diskId, mountPoint)
```
### cli
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
```
After installing the package
```
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,26 +0,0 @@
#!/usr/bin/env node
import Disposable from 'promise-toolbox/Disposable'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { mount } from './index.mjs'
async function* main([remoteUrl, vhdPathInRemote, mountPoint]) {
if (mountPoint === undefined) {
throw new TypeError('missing arg: cli <remoteUrl> <vhdPathInRemote> <mountPoint>')
}
const handler = yield getSyncedHandler({ url: remoteUrl })
const mounted = await mount(handler, vhdPathInRemote, mountPoint)
let disposePromise
process.on('SIGINT', async () => {
// ensure single dispose
if (!disposePromise) {
disposePromise = mounted.dispose()
}
await disposePromise
process.exit()
})
}
Disposable.wrap(main)(process.argv.slice(2))

View File

@@ -58,7 +58,7 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
},
})
return new Disposable(
() => fromCallback(cb => fuse.unmount(cb)),
fromCallback(cb => fuse.mount(cb))
() => fromCallback(() => fuse.unmount()),
fromCallback(() => fuse.mount())
)
})

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/fuse-vhd",
"version": "2.1.0",
"version": "2.0.0",
"license": "ISC",
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,14 +19,10 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.1.6",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.9.2"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"
"vhd-lib": "^4.5.0"
},
"scripts": {
"postversion": "npm publish --access public"

View File

@@ -17,14 +17,4 @@ map.get(['foo', 'bar']) // 2
map.get(['bar', 'foo']) // 3
map.get([OBJ]) // 4
map.get([{}]) // undefined
map.delete([])
for (const [key, value] of map.entries() {
console.log(key, value)
}
for (const value of map.values()) {
console.log(value)
}
```

View File

@@ -35,16 +35,6 @@ map.get(['foo', 'bar']) // 2
map.get(['bar', 'foo']) // 3
map.get([OBJ]) // 4
map.get([{}]) // undefined
map.delete([])
for (const [key, value] of map.entries() {
console.log(key, value)
}
for (const value of map.values()) {
console.log(value)
}
```
## Contributions

View File

@@ -36,31 +36,14 @@ function del(node, i, keys) {
return node
}
function* entries(node, key) {
if (node !== undefined) {
if (node instanceof Node) {
const { value } = node
if (value !== undefined) {
yield [key, node.value]
}
for (const [childKey, child] of node.children.entries()) {
yield* entries(child, key.concat(childKey))
}
} else {
yield [key, node]
}
}
}
function get(node, i, keys) {
return i === keys.length
? node instanceof Node
? node.value
: node
: node instanceof Node
? get(node.children.get(keys[i]), i + 1, keys)
: undefined
? get(node.children.get(keys[i]), i + 1, keys)
: undefined
}
function set(node, i, keys, value) {
@@ -86,22 +69,6 @@ function set(node, i, keys, value) {
return node
}
function* values(node) {
if (node !== undefined) {
if (node instanceof Node) {
const { value } = node
if (value !== undefined) {
yield node.value
}
for (const child of node.children.values()) {
yield* values(child)
}
} else {
yield node
}
}
}
exports.MultiKeyMap = class MultiKeyMap {
constructor() {
// each node is either a value or a Node if it contains children
@@ -112,10 +79,6 @@ exports.MultiKeyMap = class MultiKeyMap {
this._root = del(this._root, 0, keys)
}
entries() {
return entries(this._root, [])
}
get(keys) {
return get(this._root, 0, keys)
}
@@ -123,8 +86,4 @@ exports.MultiKeyMap = class MultiKeyMap {
set(keys, value) {
this._root = set(this._root, 0, keys, value)
}
values() {
return values(this._root)
}
}

View File

@@ -19,7 +19,7 @@ describe('MultiKeyMap', () => {
// reverse composite key
['bar', 'foo'],
]
const values = keys.map(() => Math.random())
const values = keys.map(() => ({}))
// set all values first to make sure they are all stored and not only the
// last one
@@ -27,12 +27,6 @@ describe('MultiKeyMap', () => {
map.set(key, values[i])
})
assert.deepEqual(
Array.from(map.entries()),
keys.map((key, i) => [key, values[i]])
)
assert.deepEqual(Array.from(map.values()), values)
keys.forEach((key, i) => {
// copy the key to make sure the array itself is not the key
assert.strictEqual(map.get(key.slice()), values[i])

View File

@@ -18,7 +18,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.2.0",
"version": "0.1.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -1,9 +1,10 @@
import assert from 'node:assert'
import { Socket } from 'node:net'
import { connect } from 'node:tls'
import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
import { fromCallback, pRetry, pDelay, pTimeout } from 'promise-toolbox'
import { readChunkStrict } from '@vates/read-chunk'
import { createLogger } from '@xen-orchestra/log'
import {
INIT_PASSWD,
NBD_CMD_READ,
@@ -20,6 +21,7 @@ import {
OPTS_MAGIC,
NBD_CMD_DISC,
} from './constants.mjs'
const { warn } = createLogger('vates:nbd-client')
// documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -38,7 +40,6 @@ export default class NbdClient {
#readBlockRetries
#reconnectRetry
#connectTimeout
#messageTimeout
// AFAIK, there is no guaranty the server answers in the same order as the queries
// so we handle a backlog of command waiting for response and handle concurrency manually
@@ -51,14 +52,7 @@ export default class NbdClient {
#reconnectingPromise
constructor(
{ address, port = NBD_DEFAULT_PORT, exportname, cert },
{
connectTimeout = 6e4,
messageTimeout = 6e4,
waitBeforeReconnect = 1e3,
readAhead = 10,
readBlockRetries = 5,
reconnectRetry = 5,
} = {}
{ connectTimeout = 6e4, waitBeforeReconnect = 1e3, readAhead = 10, readBlockRetries = 5, reconnectRetry = 5 } = {}
) {
this.#serverAddress = address
this.#serverPort = port
@@ -69,7 +63,6 @@ export default class NbdClient {
this.#readBlockRetries = readBlockRetries
this.#reconnectRetry = reconnectRetry
this.#connectTimeout = connectTimeout
this.#messageTimeout = messageTimeout
}
get exportSize() {
@@ -122,27 +115,13 @@ export default class NbdClient {
if (!this.#connected) {
return
}
this.#connected = false
const socket = this.#serverSocket
const queryId = this.#nextCommandQueryId
this.#nextCommandQueryId++
const buffer = Buffer.alloc(28)
buffer.writeInt32BE(NBD_REQUEST_MAGIC, 0) // it is a nbd request
buffer.writeInt16BE(0, 4) // no command flags for a disconnect
buffer.writeInt16BE(NBD_CMD_DISC, 6) // we want to disconnect from nbd server
buffer.writeBigUInt64BE(queryId, 8)
buffer.writeBigUInt64BE(0n, 16)
buffer.writeInt32BE(0, 24)
const promise = pFromCallback(cb => {
socket.end(buffer, 'utf8', cb)
})
try {
await pTimeout.call(promise, this.#messageTimeout)
} catch (error) {
socket.destroy()
}
await this.#write(buffer)
await this.#serverSocket.destroy()
this.#serverSocket = undefined
this.#connected = false
}
@@ -216,13 +195,11 @@ export default class NbdClient {
}
#read(length) {
const promise = readChunkStrict(this.#serverSocket, length)
return pTimeout.call(promise, this.#messageTimeout)
return readChunkStrict(this.#serverSocket, length)
}
#write(buffer) {
const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
return pTimeout.call(promise, this.#messageTimeout)
return fromCallback.call(this.#serverSocket, 'write', buffer)
}
async #readInt32() {
@@ -255,20 +232,19 @@ export default class NbdClient {
}
try {
this.#waitingForResponse = true
const buffer = await this.#read(16)
const magic = buffer.readInt32BE(0)
const magic = await this.#readInt32()
if (magic !== NBD_REPLY_MAGIC) {
throw new Error(`magic number for block answer is wrong : ${magic} ${NBD_REPLY_MAGIC}`)
}
const error = buffer.readInt32BE(4)
const error = await this.#readInt32()
if (error !== 0) {
// @todo use error code from constants.mjs
throw new Error(`GOT ERROR CODE : ${error}`)
}
const blockQueryId = buffer.readBigUInt64BE(8)
const blockQueryId = await this.#readInt64()
const query = this.#commandQueryBacklog.get(blockQueryId)
if (!query) {
throw new Error(` no query associated with id ${blockQueryId}`)
@@ -289,7 +265,7 @@ export default class NbdClient {
}
}
async #readBlock(index, size) {
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
// we don't want to add anything in backlog while reconnecting
if (this.#reconnectingPromise) {
await this.#reconnectingPromise
@@ -305,13 +281,7 @@ export default class NbdClient {
buffer.writeInt16BE(NBD_CMD_READ, 6) // we want to read a data block
buffer.writeBigUInt64BE(queryId, 8)
// byte offset in the raw disk
const offset = BigInt(index) * BigInt(size)
const remaining = this.#exportSize - offset
if (remaining < BigInt(size)) {
size = Number(remaining)
}
buffer.writeBigUInt64BE(offset, 16)
buffer.writeBigUInt64BE(BigInt(index) * BigInt(size), 16)
buffer.writeInt32BE(size, 24)
return new Promise((resolve, reject) => {
@@ -337,13 +307,45 @@ export default class NbdClient {
})
}
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
return pRetry(() => this.#readBlock(index, size), {
tries: this.#readBlockRetries,
onRetry: async err => {
warn('will retry reading block ', index, err)
await this.reconnect()
},
})
async *readBlocks(indexGenerator) {
// default : read all blocks
if (indexGenerator === undefined) {
const exportSize = this.#exportSize
const chunkSize = 2 * 1024 * 1024
indexGenerator = function* () {
const nbBlocks = Math.ceil(Number(exportSize / BigInt(chunkSize)))
for (let index = 0; BigInt(index) < nbBlocks; index++) {
yield { index, size: chunkSize }
}
}
}
const readAhead = []
const readAheadMaxLength = this.#readAhead
const makeReadBlockPromise = (index, size) => {
const promise = pRetry(() => this.readBlock(index, size), {
tries: this.#readBlockRetries,
onRetry: async err => {
warn('will retry reading block ', index, err)
await this.reconnect()
},
})
// error is handled during unshift
promise.catch(() => {})
return promise
}
// read all blocks, but try to keep readAheadMaxLength promise waiting ahead
for (const { index, size } of indexGenerator()) {
// stack readAheadMaxLength promises before starting to handle the results
if (readAhead.length === readAheadMaxLength) {
// any error will stop reading blocks
yield readAhead.shift()
}
readAhead.push(makeReadBlockPromise(index, size))
}
while (readAhead.length > 0) {
yield readAhead.shift()
}
}
}

View File

@@ -1,85 +0,0 @@
import { asyncEach } from '@vates/async-each'
import { NBD_DEFAULT_BLOCK_SIZE } from './constants.mjs'
import NbdClient from './index.mjs'
import { createLogger } from '@xen-orchestra/log'
const { warn } = createLogger('vates:nbd-client:multi')
export default class MultiNbdClient {
#clients = []
#readAhead
get exportSize() {
return this.#clients[0].exportSize
}
constructor(settings, { nbdConcurrency = 8, readAhead = 16, ...options } = {}) {
this.#readAhead = readAhead
if (!Array.isArray(settings)) {
settings = [settings]
}
for (let i = 0; i < nbdConcurrency; i++) {
this.#clients.push(
new NbdClient(settings[i % settings.length], { ...options, readAhead: Math.ceil(readAhead / nbdConcurrency) })
)
}
}
async connect() {
const connectedClients = []
for (const clientId in this.#clients) {
const client = this.#clients[clientId]
try {
await client.connect()
connectedClients.push(client)
} catch (err) {
client.disconnect().catch(() => {})
warn(`can't connect to one nbd client`, { err })
}
}
if (connectedClients.length === 0) {
throw new Error(`Fail to connect to any Nbd client`)
}
if (connectedClients.length < this.#clients.length) {
warn(
`incomplete connection by multi Nbd, only ${connectedClients.length} over ${this.#clients.length} expected clients`
)
this.#clients = connectedClients
}
}
async disconnect() {
await asyncEach(this.#clients, client => client.disconnect(), {
stopOnError: false,
})
}
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
const clientId = index % this.#clients.length
return this.#clients[clientId].readBlock(index, size)
}
async *readBlocks(indexGenerator) {
// default : read all blocks
const readAhead = []
const makeReadBlockPromise = (index, size) => {
const promise = this.readBlock(index, size)
// error is handled during unshift
promise.catch(() => {})
return promise
}
// read all blocks, but try to keep readAheadMaxLength promise waiting ahead
for (const { index, size } of indexGenerator()) {
// stack readAheadMaxLength promises before starting to handle the results
if (readAhead.length === this.#readAhead) {
// any error will stop reading blocks
yield readAhead.shift()
}
readAhead.push(makeReadBlockPromise(index, size))
}
while (readAhead.length > 0) {
yield readAhead.shift()
}
}
}

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.0.1",
"version": "2.0.0",
"engines": {
"node": ">=14.0"
},
@@ -24,14 +24,14 @@
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.6.0",
"promise-toolbox": "^0.21.0",
"xen-api": "^3.0.1"
"xen-api": "^1.3.4"
},
"devDependencies": {
"tap": "^18.7.0",
"tap": "^16.3.0",
"tmp": "^0.2.1"
},
"scripts": {
"postversion": "npm publish --access public",
"test-integration": "tap --allow-incomplete-coverage"
"test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
}
}

View File

@@ -1,3 +1,4 @@
import NbdClient from '../index.mjs'
import { spawn, exec } from 'node:child_process'
import fs from 'node:fs/promises'
import { test } from 'tap'
@@ -6,10 +7,8 @@ import { pFromCallback } from 'promise-toolbox'
import { Socket } from 'node:net'
import { NBD_DEFAULT_PORT } from '../constants.mjs'
import assert from 'node:assert'
import MultiNbdClient from '../multi.mjs'
const CHUNK_SIZE = 1024 * 1024 // non default size
const FILE_SIZE = 1024 * 1024 * 9.5 // non aligned file size
const FILE_SIZE = 10 * 1024 * 1024
async function createTempFile(size) {
const tmpPath = await pFromCallback(cb => tmp.file(cb))
@@ -82,7 +81,7 @@ test('it works with unsecured network', async tap => {
const path = await createTempFile(FILE_SIZE)
let nbdServer = await spawnNbdKit(path)
const client = new MultiNbdClient(
const client = new NbdClient(
{
address: '127.0.0.1',
exportname: 'MY_SECRET_EXPORT',
@@ -110,13 +109,13 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
`,
},
{
nbdConcurrency: 1,
readAhead: 2,
}
)
await client.connect()
tap.equal(client.exportSize, BigInt(FILE_SIZE))
const CHUNK_SIZE = 1024 * 1024 // non default size
const indexes = []
for (let i = 0; i < FILE_SIZE / CHUNK_SIZE; i++) {
indexes.push(i)
@@ -128,9 +127,9 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
})
let i = 0
for await (const block of nbdIterator) {
let blockOk = block.length === Math.min(CHUNK_SIZE, FILE_SIZE - CHUNK_SIZE * i)
let blockOk = true
let firstFail
for (let j = 0; j < block.length; j += 4) {
for (let j = 0; j < CHUNK_SIZE; j += 4) {
const wanted = i * CHUNK_SIZE + j
const found = block.readUInt32BE(j)
blockOk = blockOk && found === wanted
@@ -138,7 +137,7 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
firstFail = j
}
}
tap.ok(blockOk, `check block ${i} content ${block.length}`)
tap.ok(blockOk, `check block ${i} content`)
i++
// flaky server is flaky
@@ -148,6 +147,17 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
nbdServer = await spawnNbdKit(path)
}
}
// we can reuse the conneciton to read other blocks
// default iterator
const nbdIteratorWithDefaultBlockIterator = client.readBlocks()
let nb = 0
for await (const block of nbdIteratorWithDefaultBlockIterator) {
nb++
tap.equal(block.length, 2 * 1024 * 1024)
}
tap.equal(nb, 5)
assert.rejects(() => client.readBlock(100, CHUNK_SIZE))
await client.disconnect()

View File

@@ -31,20 +31,14 @@ export function Client(vCenterHostname, username, password, sslVerify) {
EventEmitter.call(this)
// sslVerify argument handling
// don't use any proxy to connect to the esxi
if (sslVerify) {
this.clientopts = {
request: axios.create({
proxy: false,
}),
}
this.clientopts = {}
} else {
this.clientopts = {
request: axios.create({
httpsAgent: new https.Agent({
rejectUnauthorized: false,
}),
proxy: false,
}),
}
}

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/node-vsphere-soap",
"version": "2.0.1",
"version": "2.0.0",
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
"main": "lib/client.mjs",
"author": "reedog117",

View File

@@ -9,15 +9,6 @@ function assert(name, value) {
}
}
const WHITESPACES = /\s+/g
function cleanToken(otp) {
// remove whitespaces
//
// some OTP clients add whitespaces for better readability (e.g. `XXX XXX`)
// and users might be tempted to write them
return otp && otp.replace(WHITESPACES, '')
}
// https://github.com/google/google-authenticator/wiki/Key-Uri-Format
function generateUri(protocol, label, params) {
assert('label', typeof label === 'string')
@@ -73,7 +64,7 @@ export function generateHotpUri({ counter, digits, issuer, label, secret }) {
}
export async function verifyHotp(token, opts) {
return cleanToken(token) === (await generateHotp(opts))
return token === (await generateHotp(opts))
}
function totpCounter(period = 30, timestamp = Math.floor(Date.now() / 1e3)) {
@@ -94,7 +85,6 @@ export async function verifyTotp(token, { period, timestamp, window = 1, ...opts
const counter = totpCounter(period, timestamp)
const end = counter + window
opts.counter = counter - window
token = cleanToken(token)
while (opts.counter <= end) {
if (token === (await generateHotp(opts))) {
return true

View File

@@ -1,5 +1,5 @@
import { strict as assert } from 'node:assert'
import test from 'test'
import { describe, it } from 'tap/mocha'
import {
generateHotp,
@@ -11,8 +11,6 @@ import {
verifyTotp,
} from './index.mjs'
const { describe, it } = test
describe('generateSecret', function () {
it('generates a string of 32 chars', async function () {
const secret = generateSecret()

View File

@@ -22,7 +22,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "1.1.0",
"version": "1.0.0",
"engines": {
"node": ">=15"
},
@@ -31,9 +31,9 @@
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.3.0"
}
}

View File

@@ -1,7 +1,7 @@
'use strict'
const assert = require('assert/strict')
const { describe, it } = require('test')
const { describe, it } = require('tap').mocha
const { every, not, some } = require('./')

View File

@@ -32,9 +32,9 @@
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.0.1"
}
}

View File

@@ -22,41 +22,41 @@ const readChunk = (stream, size) =>
stream.errored != null
? Promise.reject(stream.errored)
: stream.closed || stream.readableEnded
? Promise.resolve(null)
: new Promise((resolve, reject) => {
if (size !== undefined) {
assert(size > 0)
? Promise.resolve(null)
: new Promise((resolve, reject) => {
if (size !== undefined) {
assert(size > 0)
// per Node documentation:
// > The size argument must be less than or equal to 1 GiB.
assert(size < 1073741824)
}
// per Node documentation:
// > The size argument must be less than or equal to 1 GiB.
assert(size < 1073741824)
}
function onEnd() {
resolve(null)
function onEnd() {
resolve(null)
removeListeners()
}
function onError(error) {
reject(error)
removeListeners()
}
function onReadable() {
const data = stream.read(size)
if (data !== null) {
resolve(data)
removeListeners()
}
function onError(error) {
reject(error)
removeListeners()
}
function onReadable() {
const data = stream.read(size)
if (data !== null) {
resolve(data)
removeListeners()
}
}
function removeListeners() {
stream.removeListener('end', onEnd)
stream.removeListener('error', onError)
stream.removeListener('readable', onReadable)
}
stream.on('end', onEnd)
stream.on('error', onError)
stream.on('readable', onReadable)
onReadable()
})
}
function removeListeners() {
stream.removeListener('end', onEnd)
stream.removeListener('error', onError)
stream.removeListener('readable', onReadable)
}
stream.on('end', onEnd)
stream.on('error', onError)
stream.on('readable', onReadable)
onReadable()
})
exports.readChunk = readChunk
/**
@@ -111,42 +111,42 @@ async function skip(stream, size) {
return stream.errored != null
? Promise.reject(stream.errored)
: size === 0 || stream.closed || stream.readableEnded
? Promise.resolve(0)
: new Promise((resolve, reject) => {
let left = size
function onEnd() {
resolve(size - left)
removeListeners()
}
function onError(error) {
reject(error)
removeListeners()
}
function onReadable() {
const data = stream.read()
left -= data === null ? 0 : data.length
if (left > 0) {
// continue to read
} else {
// if more than wanted has been read, push back the rest
if (left < 0) {
stream.unshift(data.slice(left))
}
resolve(size)
removeListeners()
? Promise.resolve(0)
: new Promise((resolve, reject) => {
let left = size
function onEnd() {
resolve(size - left)
removeListeners()
}
function onError(error) {
reject(error)
removeListeners()
}
function onReadable() {
const data = stream.read()
left -= data === null ? 0 : data.length
if (left > 0) {
// continue to read
} else {
// if more than wanted has been read, push back the rest
if (left < 0) {
stream.unshift(data.slice(left))
}
resolve(size)
removeListeners()
}
function removeListeners() {
stream.removeListener('end', onEnd)
stream.removeListener('error', onError)
stream.removeListener('readable', onReadable)
}
stream.on('end', onEnd)
stream.on('error', onError)
stream.on('readable', onReadable)
onReadable()
})
}
function removeListeners() {
stream.removeListener('end', onEnd)
stream.removeListener('error', onError)
stream.removeListener('readable', onReadable)
}
stream.on('end', onEnd)
stream.on('error', onError)
stream.on('readable', onReadable)
onReadable()
})
}
exports.skip = skip

View File

@@ -27,7 +27,7 @@
"license": "ISC",
"version": "0.1.0",
"engines": {
"node": ">=12.3"
"node": ">=10"
},
"scripts": {
"postversion": "npm publish --access public",

View File

@@ -72,27 +72,6 @@ Task.warning(message, data)
Task.set(property, value)
```
### `Task.run([opts], fn, ...args)`
This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
```js
// options can be passed
await Task.run({ properties: { name: 'foo' } }, fn)
// arguments can be passed
await Task.run(fn, 'foo', 'bar')
// context (this) is forwarded
await Task.run.call(object, object.method)
// as a convenience, you can pass a method name directly
await Task.run.call(object, 'method')
// everything together
await Task.run.call(object, 'method', 'foo', 'bar')
```
### `combineEvents`
Create a consolidated log from individual events.
@@ -144,7 +123,7 @@ const onProgress = makeOnProgress({
onTaskUpdate(taskLog) {},
})
Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
```
It can also be fed event logs directly:

View File

@@ -88,27 +88,6 @@ Task.warning(message, data)
Task.set(property, value)
```
### `Task.run([opts], fn, ...args)`
This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
```js
// options can be passed
await Task.run({ properties: { name: 'foo' } }, fn)
// arguments can be passed
await Task.run(fn, 'foo', 'bar')
// context (this) is forwarded
await Task.run.call(object, object.method)
// as a convenience, you can pass a method name directly
await Task.run.call(object, 'method')
// everything together
await Task.run.call(object, 'method', 'foo', 'bar')
```
### `combineEvents`
Create a consolidated log from individual events.
@@ -132,7 +111,7 @@ const onProgress = makeOnProgress({
// current status of the task as described in the previous section
taskLog.status
// undefined or a dictionary of properties attached to the task
// undefined or a dictionnary of properties attached to the task
taskLog.properties
// timestamp at which the abortion was requested, undefined otherwise
@@ -160,7 +139,7 @@ const onProgress = makeOnProgress({
onTaskUpdate(taskLog) {},
})
Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
```
It can also be fed event logs directly:

View File

@@ -36,25 +36,8 @@ exports.Task = class Task {
}
}
static run(...args) {
let opts = args[0]
let fn
if (typeof opts === 'object') {
args.shift()
fn = args[0]
} else {
fn = opts
opts = undefined
}
args.shift()
const thisArg = this !== Task ? this : undefined
if (typeof fn !== 'function') {
fn = this[fn]
}
return new Task(opts).run(() => fn.apply(thisArg, args))
static run(opts, fn) {
return new this(opts).run(fn)
}
static set(name, value) {

View File

@@ -3,7 +3,6 @@
const assert = require('node:assert').strict
const { describe, it } = require('test')
const { makeOnProgress } = require('./combineEvents.js')
const { Task } = require('./index.js')
const noop = Function.prototype
@@ -163,44 +162,6 @@ describe('Task', function () {
})
})
describe('.run()', function () {
function fn(...args) {
Task.set('baz', 'qux')
return [this, ...args]
}
const obj = { fn }
it('accepts optional options', async function () {
assert.deepEqual(
await Task.run(
{
properties: { foo: 'bar' },
onProgress: makeOnProgress({
onRootTaskEnd(log) {
assert.deepEqual(log.properties, { __proto__: null, foo: 'bar', baz: 'qux' })
},
}),
},
fn
),
[undefined]
)
})
it('accepts optional arguments', async function () {
assert.deepEqual(await Task.run(fn, 'foo', 'bar'), [undefined, 'foo', 'bar'])
})
it('accepts optional context', async function () {
assert.deepEqual(await Task.run.call('foo', fn), ['foo'])
})
it('accepts a method name instead of a function', async function () {
assert.deepEqual(await Task.run.call(obj, 'fn'), [obj])
})
})
describe('.set()', function () {
it('does nothing when run outside a task', function () {
Task.set('progress', 10)

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.3.0",
"version": "0.2.0",
"engines": {
"node": ">=14"
},

View File

@@ -1,128 +0,0 @@
### Formatting
```js
import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// → {
// name: 'methodCall',
// children: [
// {
// name: 'methodName',
// children: [ 'examples.getStateName' ]
// },
// {
// name: 'params',
// children: [
// {
// name: 'param',
// children: [ { name: 'value', children: [ '40' ] } ]
// }
// ]
// }
// ]
// }
// There is a dedicated method to format a JS value
xmlRpcFormatter.format_js(new Date(900684535000))
// → {
// name: 'dateTime.iso8601',
// children: [ '1998-07-17T14:08:55.000Z' ]
// }
```
Formatting XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { formatXml } from '@vates/xml/format'
// it can now be passed to the XML-RPC parser
const tree = xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
const xml = formatXml(tree, { indent: 0 })
```
### Parsing
```js
import { xmlRpcParser } from '@vates/xml-rpc/parser'
xmlRpcParser.parse({
name: 'methodResponse',
children: [
{
name: 'params',
children: [
{
name: 'param',
children: [
{
name: 'value',
children: [
{
name: 'string',
children: ['South Dakota'],
},
],
},
],
},
],
},
],
})
// { params: [ 'South Dakota' ] }
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcParser.parse_array({
name: 'array',
children: [
{
name: 'data',
children: [
{
name: 'value',
children: [{ name: 'boolean', children: ['1'] }],
},
{
name: 'value',
children: [{ name: 'int', children: ['42'] }],
},
],
},
],
})
// [ true, 42 ]
```
Parsing XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { parseXml } from '@vates/xml/parse'
const xml = `
<?xml version="1.0"?>
<methodResponse>
<params>
<param>
<value><string>South Dakota</string></value>
</param>
</params>
</methodResponse>
`
const tree = parseXml(xml)
// it can now be passed to the XML-RPC parser
xmlRpcParser.parse(tree)
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,161 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/xml-rpc
[![Package Version](https://badgen.net/npm/v/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc) ![License](https://badgen.net/npm/license/@vates/xml-rpc) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml-rpc)](https://bundlephobia.com/result?p=@vates/xml-rpc) [![Node compatibility](https://badgen.net/npm/node/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc)
> Parse and format XML-RPC messages, do not handle transport or XML
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/xml-rpc):
```sh
npm install --save @vates/xml-rpc
```
## Usage
### Formatting
```js
import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// → {
// name: 'methodCall',
// children: [
// {
// name: 'methodName',
// children: [ 'examples.getStateName' ]
// },
// {
// name: 'params',
// children: [
// {
// name: 'param',
// children: [ { name: 'value', children: [ '40' ] } ]
// }
// ]
// }
// ]
// }
// There is a dedicated method to format a JS value
xmlRpcFormatter.format_js(new Date(900684535000))
// → {
// name: 'dateTime.iso8601',
// children: [ '1998-07-17T14:08:55.000Z' ]
// }
```
Formatting XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { formatXml } from '@vates/xml/format'
// it can now be passed to the XML-RPC parser
const tree = xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
const xml = formatXml(tree, { indent: 0 })
```
### Parsing
```js
import { xmlRpcParser } from '@vates/xml-rpc/parser'
xmlRpcParser.parse({
name: 'methodResponse',
children: [
{
name: 'params',
children: [
{
name: 'param',
children: [
{
name: 'value',
children: [
{
name: 'string',
children: ['South Dakota'],
},
],
},
],
},
],
},
],
})
// { params: [ 'South Dakota' ] }
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcParser.parse_array({
name: 'array',
children: [
{
name: 'data',
children: [
{
name: 'value',
children: [{ name: 'boolean', children: ['1'] }],
},
{
name: 'value',
children: [{ name: 'int', children: ['42'] }],
},
],
},
],
})
// [ true, 42 ]
```
Parsing XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { parseXml } from '@vates/xml/parse'
const xml = `
<?xml version="1.0"?>
<methodResponse>
<params>
<param>
<value><string>South Dakota</string></value>
</param>
</params>
</methodResponse>
`
const tree = parseXml(xml)
// it can now be passed to the XML-RPC parser
xmlRpcParser.parse(tree)
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,104 +0,0 @@
'use strict'
function node(name, attributes, children) {
if (Array.isArray(attributes)) {
children = attributes
attributes = undefined
}
return { name, attributes, children }
}
class XmlRpcFormatter {
format_js(value) {
const type = typeof value
if (type === 'object') {
return value === null
? this.format_nil()
: value instanceof Date
? this['format_dateTime.iso8601'](value)
: Array.isArray(value)
? this.format_array(value)
: Buffer.isBuffer(value)
? this.format_base64(value)
: this.format_struct(value)
}
return this['format_' + type](value)
}
format_array(values) {
return node('array', [node('data', values.map(this.format_value, this))])
}
format_base64(buf) {
return node('base64', [buf.toString('base64')])
}
format_boolean(bool) {
return node('boolean', [bool ? '1' : '0'])
}
'format_dateTime.iso8601'(date) {
return node('dateTime.iso8601', [date.toISOString()])
}
format_double(str) {
return node('double', [String(str)])
}
format_fault(val) {
return node('fault', [this.format_value(val)])
}
format_i4(str) {
return node('i4', [String(str)])
}
format_int(str) {
return node('int', [String(str)])
}
format_member([name, value]) {
return node('member', [node('name', [name]), this.format_value(value)])
}
format_methodCall({ methodName, params }) {
return node('methodCall', [node('methodName', [methodName]), this.format_params(params)])
}
format_methodResponse({ fault, params }) {
return node('methodResponse', [fault !== undefined ? this.format_fault(fault) : this.format_params(params)])
}
format_nil() {
return node('nil')
}
format_number(num) {
return Number.isInteger(num) ? this.format_int(num) : this.format_double(num)
}
format_param(value) {
return node('param', [this.format_value(value)])
}
format_params(params) {
return node('params', params.map(this.format_param, this))
}
format_string(str) {
return str
}
format_struct(struct) {
return node('struct', Object.entries(struct).map(this.format_member, this))
}
format_value(value) {
return node('value', [this.format_js(value)])
}
}
exports.Formatter = XmlRpcFormatter
exports.xmlRpcFormatter = new XmlRpcFormatter()

View File

@@ -1,33 +0,0 @@
{
"private": false,
"name": "@vates/xml-rpc",
"description": "Parse and format XML-RPC messages, do not handle transport or XML",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml-rpc",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/xml-rpc",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "1.0.0",
"engines": {
"node": ">=12"
},
"devDependencies": {
"@vates/xml": "^2.0.0",
"test": "^3.3.0"
},
"exports": {
"./formatter": "./formatter.js",
"./parser": "./parser.js"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
}
}

View File

@@ -1,133 +0,0 @@
'use strict'
function toValue(node) {
const obj = { __proto__: null }
for (const child of node.children) {
if (typeof child === 'string') {
return child
}
const { name } = child
if (Object.hasOwn(obj, name)) {
const prev = obj[name]
if (Array.isArray(obj, name)) {
prev.push(child)
} else {
obj[name] = [prev, child]
}
} else {
obj[name] = child
}
}
return obj
}
// https://en.wikipedia.org/wiki/XML-RPC#Data_types
class XmlRpcParser {
parse(node) {
return this['parse_' + node.name](node)
}
parse_array({ children: [data] }) {
return data.children.map(this.parse_value, this)
}
parse_base64({ children: [str] }) {
return Buffer.from(str, 'base64')
}
parse_boolean({ children: [str] }) {
return Boolean(Number(str))
}
'parse_dateTime.iso8601'({ children: [str] }) {
const date = new Date(str)
// date successfully parsed
if (!Number.isNaN(+date)) {
return date
}
// XML-RPC dates may use YYYYMMDD instead of YYYY-MM-DD which ECMAScript
// does not parse
return new Date(str.slice(0, 4) + '-' + str.slice(4, 6) + '-' + str.slice(6))
}
parse_double({ children: [str] }) {
return Number(str)
}
parse_fault({ children: [value] }) {
return this.parse_value(value)
}
parse_i4({ children: [str] }) {
return Number(str)
}
parse_int({ children: [str] }) {
return Number(str)
}
parse_member(member) {
member = toValue(member)
return [member.name.children[0], this.parse_value(member.value)]
}
parse_methodCall(methodCall) {
methodCall = toValue(methodCall)
return {
methodName: methodCall.methodName.children[0],
params: this.parse_params(methodCall.params),
}
}
parse_methodResponse(methodResponse) {
methodResponse = toValue(methodResponse)
const { fault } = methodResponse
if (fault !== undefined) {
return { fault: this.parse_fault(fault) }
}
return { params: this.parse_params(methodResponse.params) }
}
parse_nil() {
return null
}
parse_param(param) {
return this.parse_value(param.children[0])
}
parse_params(params) {
return params.children.map(this.parse_param, this)
}
parse_string({ children: [str] }) {
return str
}
parse_struct(struct) {
return Object.fromEntries(struct.children.map(this.parse_member, this))
}
parse_value({ children }) {
if (children.length === 0) {
return ''
}
const child = children[0]
// can be a plain string
if (typeof child === 'string') {
return child
}
return this.parse(child)
}
}
exports.XmlRpcParser = XmlRpcParser
exports.xmlRpcParser = new XmlRpcParser()

View File

@@ -1,19 +0,0 @@
'use strict'
const { formatXml } = require('@vates/xml/format')
const { join } = require('node:path')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { xmlRpcFormatter } = require('@vates/xml-rpc/formatter')
const assert = require('node:assert/strict')
const test = require('test')
const methodCall = require('./methodCall.js')
test(function () {
const xmlTree = parseXml(readFileSync(join(__dirname, 'methodCall.xml')))
const xmlRpc = xmlRpcFormatter.format_methodCall(methodCall)
assert.deepEqual(parseXml(formatXml(xmlRpc)), xmlTree)
})

View File

@@ -1,16 +0,0 @@
'use strict'
module.exports = {
methodName: 'examples.getStateName',
params: [
[1404, 'Something here'],
Buffer.from([121, 111, 117, 32, 99, 97, 110, 39, 116, 32, 114, 101, 97, 100, 32, 116, 104, 105, 115, 33]),
true,
new Date('1998-07-17T14:08:55.000Z'),
-12.53,
42,
'Hello world!',
{ foo: 1, bar: 2 },
null,
],
}

View File

@@ -1,73 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<methodCall>
<methodName>examples.getStateName</methodName>
<params>
<param>
<value>
<array>
<data>
<value>
<int>1404</int>
</value>
<value>
Something here
</value>
</data>
</array>
</value>
</param>
<param>
<value>
<base64>eW91IGNhbid0IHJlYWQgdGhpcyE=</base64>
</value>
</param>
<param>
<value>
<boolean>1</boolean>
</value>
</param>
<param>
<value>
<dateTime.iso8601>1998-07-17T14:08:55.000Z</dateTime.iso8601>
</value>
</param>
<param>
<value>
<double>-12.53</double>
</value>
</param>
<param>
<value>
<int>42</int>
</value>
</param>
<param>
<value>
Hello world!
</value>
</param>
<param>
<value>
<struct>
<member>
<name>foo</name>
<value>
<int>1</int>
</value>
</member>
<member>
<name>bar</name>
<value>
<int>2</int>
</value>
</member>
</struct>
</value>
</param>
<param>
<value>
<nil />
</value>
</param>
</params>
</methodCall>

View File

@@ -1,32 +0,0 @@
'use strict'
const { join } = require('node:path')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { xmlRpcParser } = require('@vates/xml-rpc/parser')
const assert = require('node:assert/strict')
const test = require('test')
const methodCall = require('./methodCall.js')
test('parser.parse()', function () {
const xml = String(readFileSync(join(__dirname, 'methodCall.xml')))
const xmlRpc = parseXml(xml)
assert.deepEqual(xmlRpcParser.parse(xmlRpc), methodCall)
})
test('datetime without dashes nor milliseconds', function () {
assert.deepEqual(
xmlRpcParser.parse({ name: 'dateTime.iso8601', children: ['19980717T14:08:55Z'] }),
new Date('1998-07-17T14:08:55Z')
)
})
test('i4 support', function () {
assert.equal(xmlRpcParser.parse({ name: 'i4', children: ['42'] }), 42)
})
test('string support', function () {
assert.equal(xmlRpcParser.parse({ name: 'string', children: ['foo'] }), 'foo')
})

View File

@@ -1,56 +0,0 @@
### `parseXml(xml, [opts])`
> Based on [`sax`](https://www.npmjs.com/package/sax)
`opts` is an optional object which can contain the following options:
- `normalize = true`: if true, then turn any whitespace into a single space
- `strict = true`: whether or not to be a jerk
- `trim = true`: whether or not to trim text nodes
```js
import { parseXml } from '@xen-orchestra/xml/parse'
const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>
`)
// → {
// name: 'tag1',
// attributes: { attr1: 'value1', attr2: 'value2' },
// children: [
// 'Text & entities',
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'ns:tag3', attributes: {}, children: [] }
// ]
// }
```
### `formatXml(tree, [opts])`
`opts` is an optional object which can contain the following options:
- `includeDeclaration = true`: whether to include an XML declaration
- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
```js
import { formatXml } from '@xen-orchestra/xml/format'
formatXml({
name: 'tag1',
attributes: { attr1: 'value1', attr2: 'value2' },
children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
})
// → <?xml version="1.0" encoding="UTF-8"?>
// <tag1 attr1="value1" attr2="value2">
// Text &amp; entities
// <tag2 />
// <tag2 />
// <ns:tag3 />
// </tag1>
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,89 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/xml
[![Package Version](https://badgen.net/npm/v/@vates/xml)](https://npmjs.org/package/@vates/xml) ![License](https://badgen.net/npm/license/@vates/xml) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml)](https://bundlephobia.com/result?p=@vates/xml) [![Node compatibility](https://badgen.net/npm/node/@vates/xml)](https://npmjs.org/package/@vates/xml)
> Simple XML formatting/parsing
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/xml):
```sh
npm install --save @vates/xml
```
## Usage
### `parseXml(xml, [opts])`
> Based on [`sax`](https://www.npmjs.com/package/sax)
`opts` is an optional object which can contain the following options:
- `normalize = true`: if true, then turn any whitespace into a single space
- `strict = true`: whether or not to be a jerk
- `trim = true`: whether or not to trim text nodes
```js
import { parseXml } from '@xen-orchestra/xml/parse'
const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>
`)
// → {
// name: 'tag1',
// attributes: { attr1: 'value1', attr2: 'value2' },
// children: [
// 'Text & entities',
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'ns:tag3', attributes: {}, children: [] }
// ]
// }
```
### `formatXml(tree, [opts])`
`opts` is an optional object which can contain the following options:
- `includeDeclaration = true`: whether to include an XML declaration
- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
```js
import { formatXml } from '@xen-orchestra/xml/format'
formatXml({
name: 'tag1',
attributes: { attr1: 'value1', attr2: 'value2' },
children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
})
// → <?xml version="1.0" encoding="UTF-8"?>
// <tag1 attr1="value1" attr2="value2">
// Text &amp; entities
// <tag2 />
// <tag2 />
// <ns:tag3 />
// </tag1>
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,30 +0,0 @@
#!/usr/bin/env node
'use strict'
const { inspect } = require('node:util')
const { formatXml } = require('@vates/xml/format')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
function log(val) {
process.stdout.write(inspect(val, false, null, true))
process.stdout.write('\n')
}
function main([inputPath = 0]) {
const input = readFileSync(inputPath)
// attempt to parse from JSON
let data
try {
data = JSON.parse(input)
} catch (error) {
// fallback to XML
log(parseXml(input))
return
}
process.stdout.write(formatXml(data))
}
main(process.argv.slice(2))

View File

@@ -1,64 +0,0 @@
'use strict'
const ENTITIES = {
'"': '&quot;',
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
"'": '&apos;',
}
const RE = new RegExp(Object.keys(ENTITIES).join('|'), 'g')
function replace(match) {
return ENTITIES[match]
}
function escape(str) {
return str.replace(RE, replace)
}
function formatNode(node, depth) {
const indent = this.indent.repeat(depth)
if (typeof node === 'object') {
const line = [indent, '<', node.name]
const { attributes } = node
if (attributes !== undefined) {
for (const name of Object.keys(attributes)) {
line.push(' ', name, '="', escape(attributes[name]), '"')
}
}
const { children } = node
if (children === undefined || children.length === 0) {
line.push(' />')
this.lines.push(line.join(''))
} else {
line.push('>')
this.lines.push(line.join(''))
for (const child of children) {
formatNode.call(this, child, depth + 1)
}
this.lines.push(indent + '</' + node.name + '>')
}
} else {
// string (or scalar)
this.lines.push(indent + escape(String(node)))
}
}
exports.formatXml = function formatXml(tree, { includeDeclaration = true, indent = 2 } = {}) {
const lines = []
if (includeDeclaration) {
lines.push('<?xml version="1.0" encoding="UTF-8"?>')
}
const ctx = {
lines,
indent: typeof indent === 'number' ? ' '.repeat(indent) : indent,
}
formatNode.call(ctx, tree, 0)
return lines.join(indent !== 0 && indent !== '' ? '\n' : '')
}

View File

@@ -1,35 +0,0 @@
{
"private": false,
"name": "@vates/xml",
"description": "Simple XML formatting/parsing",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/xml",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "2.0.0",
"engines": {
"node": ">=14"
},
"dependencies": {
"sax": "^1.3.0"
},
"devDependencies": {
"test": "^3.3.0"
},
"exports": {
"./format": "./format.js",
"./parse": "./parse.js"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
}
}

View File

@@ -1,23 +0,0 @@
'use strict'
const sax = require('sax')
exports.parseXml = function parseXml(xml, { normalize = true, strict = true, trim = true } = {}) {
const stack = [{ children: [] }]
const parser = sax.parser(strict, { normalize, trim })
parser.ontext = text => {
stack[stack.length - 1].children.push(text)
}
parser.onopentag = ({ name, attributes }) => {
stack.push({ name, attributes, children: [] })
}
parser.onclosetag = () => {
const node = stack.pop()
stack[stack.length - 1].children.push(node)
}
parser.write(xml).close()
return stack[0].children[0]
}

View File

@@ -1,25 +0,0 @@
{
"name": "tag1",
"attributes": {
"attr1": "value1",
"attr2": "value2"
},
"children": [
"Text & entities",
{
"name": "tag2",
"attributes": {},
"children": []
},
{
"name": "tag2",
"attributes": {},
"children": []
},
{
"name": "ns:tag3",
"attributes": {},
"children": []
}
]
}

View File

@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>

View File

@@ -1,18 +0,0 @@
'use strict'
const { formatXml } = require('@vates/xml/format')
const { join } = require('node:path')
const { readFileSync } = require('node:fs')
const assert = require('node:assert/strict')
const test = require('test')
test('format()', function () {
const xml = String(readFileSync(join(__dirname, 'data.xml')))
const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
assert.equal(formatXml(tree) + '\n', xml)
})
test('supports missing attributes and children', function () {
assert.equal(formatXml({ name: 'foo' }, { includeDeclaration: false }), '<foo />')
})

View File

@@ -1,14 +0,0 @@
'use strict'
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { join } = require('node:path')
const assert = require('node:assert/strict')
const test = require('test')
test('parse()', function () {
const xml = String(readFileSync(join(__dirname, 'data.xml')))
const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
assert.deepEqual(parseXml(xml), tree)
})

View File

@@ -35,7 +35,7 @@
"test": "node--test"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^15.0.1",
"test": "^3.2.1"
}
}

View File

@@ -130,10 +130,6 @@ class AuditCore {
return nValid
}
get(id) {
return this._storage.get(id)
}
async *getFrom(newest) {
const storage = this._storage

View File

@@ -1,7 +1,7 @@
'use strict'
const assert = require('assert/strict')
const { afterEach, describe, it } = require('test')
const { afterEach, describe, it } = require('tap').mocha
const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')

View File

@@ -7,16 +7,16 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.3.0",
"version": "0.2.3",
"engines": {
"node": ">=14"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap --lines 67 --functions 92 --branches 52 --statements 67"
},
"dependencies": {
"@vates/decorate-with": "^2.1.0",
"@vates/decorate-with": "^2.0.0",
"@xen-orchestra/log": "^0.6.0",
"golike-defer": "^0.5.1",
"object-hash": "^2.0.1"
@@ -28,6 +28,6 @@
"url": "https://vates.fr"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.0.1"
}
}

View File

@@ -7,8 +7,8 @@
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.48.1",
"@xen-orchestra/fs": "^4.1.6",
"@xen-orchestra/backups": "^0.40.0",
"@xen-orchestra/fs": "^4.0.1",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",
"lodash": "^4.17.15",
@@ -27,7 +27,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "1.0.18",
"version": "1.0.10",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -4,229 +4,23 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { importIncrementalVm } from './_incrementalVm.mjs'
import { Task } from './Task.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { VhdNegative, VhdSynthetic } from 'vhd-lib'
import { decorateClass } from '@vates/decorate-with'
import { createLogger } from '@xen-orchestra/log'
import { dirname, join } from 'node:path'
import pickBy from 'lodash/pickBy.js'
import { defer } from 'golike-defer'
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
async function resolveUuid(xapi, cache, uuid, type) {
if (uuid == null) {
return uuid
}
const ref = cache.get(uuid)
if (ref === undefined) {
cache.set(uuid, xapi.call(`${type}.get_by_uuid`, uuid))
}
return cache.get(uuid)
}
export class ImportVmBackup {
constructor({
adapter,
metadata,
srUuid,
xapi,
settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
}) {
constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs = {} } = {} }) {
this._adapter = adapter
this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
this._importIncrementalVmSettings = { newMacAddresses, mapVdisSrs }
this._metadata = metadata
this._srUuid = srUuid
this._xapi = xapi
}
async #getPathOfVdiSnapshot(snapshotUuid) {
const metadata = this._metadata
if (this._pathToVdis === undefined) {
const backups = await this._adapter.listVmBackups(
this._metadata.vm.uuid,
({ mode, timestamp }) => mode === 'delta' && timestamp >= metadata.timestamp
)
const map = new Map()
for (const backup of backups) {
for (const [vdiRef, vdi] of Object.entries(backup.vdis)) {
map.set(vdi.uuid, backup.vhds[vdiRef])
}
}
this._pathToVdis = map
}
return this._pathToVdis.get(snapshotUuid)
}
async _reuseNearestSnapshot($defer, ignoredVdis) {
const metadata = this._metadata
const { mapVdisSrs } = this._importIncrementalVmSettings
const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
const streams = {}
const metdataDir = dirname(metadata._filename)
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
for (const [vdiRef, vdi] of Object.entries(vdis)) {
const vhdPath = join(metdataDir, vhds[vdiRef])
let xapiDisk
try {
xapiDisk = await this._xapi.getRecordByUuid('VDI', vdi.$snapshot_of$uuid)
} catch (err) {
// if this disk is not present anymore, fall back to default restore
warn(err)
}
let snapshotCandidate, backupCandidate
if (xapiDisk !== undefined) {
debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
for (const snapshotRef of xapiDisk.snapshots) {
const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
debug('handling snapshot', { snapshot })
// take only the first snapshot
if (snapshotCandidate && snapshotCandidate.snapshot_time < snapshot.snapshot_time) {
debug('already got a better candidate')
continue
}
// have a corresponding backup more recent than metadata ?
const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
if (pathToSnapshotData === undefined) {
debug('no backup linked to this snaphot')
continue
}
if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
debug('not restored on the same SR', { snapshotSr: snapshot.$SR.uuid, mapVdisSrs, srUuid: this._srUuid })
continue
}
debug('got a candidate', pathToSnapshotData)
snapshotCandidate = snapshot
backupCandidate = pathToSnapshotData
}
}
let stream
const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
if (vhdPath === backupWithSnapshotPath) {
// all the data are already on the host
debug('direct reuse of a snapshot')
stream = null
vdis[vdiRef].baseVdi = snapshotCandidate
// go next disk , we won't use this stream
continue
}
let disposableDescendants
const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
// this will also clean if another disk of this VM backup fails
// if user really only need to restore non failing disks he can retry with ignoredVdis
let disposed = false
const disposeOnce = async () => {
if (!disposed) {
disposed = true
try {
await disposableDescendants?.dispose()
await disposableSynthetic?.dispose()
} catch (error) {
warn('openVhd: failed to dispose VHDs', { error })
}
}
}
$defer.onFailure(() => disposeOnce())
const parentVhd = disposableSynthetic.value
await parentVhd.readBlockAllocationTable()
debug('got vhd synthetic of parents', parentVhd.length)
if (snapshotCandidate !== undefined) {
try {
debug('will try to use differential restore', {
backupWithSnapshotPath,
vhdPath,
vdiRef,
})
disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
until: vhdPath,
})
const descendantsVhd = disposableDescendants.value
await descendantsVhd.readBlockAllocationTable()
debug('got vhd synthetic of descendants')
const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
debug('got vhd negative')
// update the stream with the negative vhd stream
stream = await negativeVhd.stream()
vdis[vdiRef].baseVdi = snapshotCandidate
} catch (error) {
// can be a broken VHD chain, a vhd chain with a key backup, ....
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
warn(`can't use differential restore`, { error })
disposableDescendants?.dispose()
}
}
// didn't make a negative stream : fallback to classic stream
if (stream === undefined) {
debug('use legacy restore')
stream = await parentVhd.stream()
}
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
info('everything is ready, will transfer', stream.length)
streams[`${vdiRef}.vhd`] = stream
}
return {
streams,
vbds,
vdis,
version: '1.0.0',
vifs,
vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
}
}
async #decorateIncrementalVmMetadata() {
const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
const ignoredVdis = new Set(
Object.entries(mapVdisSrs)
.filter(([_, srUuid]) => srUuid === null)
.map(([vdiUuid]) => vdiUuid)
)
let backup
if (useDifferentialRestore) {
backup = await this._reuseNearestSnapshot(ignoredVdis)
} else {
backup = await this._adapter.readIncrementalVmBackup(this._metadata, ignoredVdis)
}
const xapi = this._xapi
const cache = new Map()
const mapVdisSrRefs = {}
if (additionnalVmTag !== undefined) {
backup.vm.tags.push(additionnalVmTag)
}
for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
}
const srRef = await resolveUuid(xapi, cache, this._srUuid, 'SR')
Object.values(backup.vdis).forEach(vdi => {
vdi.SR = mapVdisSrRefs[vdi.uuid] ?? srRef
})
return backup
}
async run() {
const adapter = this._adapter
const metadata = this._metadata
const isFull = metadata.mode === 'full'
const sizeContainer = { size: 0 }
const { newMacAddresses } = this._importIncrementalVmSettings
let backup
if (isFull) {
backup = await adapter.readFullVmBackup(metadata)
@@ -234,7 +28,12 @@ export class ImportVmBackup {
} else {
assert.strictEqual(metadata.mode, 'delta')
backup = await this.#decorateIncrementalVmMetadata()
const ignoredVdis = new Set(
Object.entries(this._importIncrementalVmSettings.mapVdisSrs)
.filter(([_, srUuid]) => srUuid === null)
.map(([vdiUuid]) => vdiUuid)
)
backup = await adapter.readIncrementalVmBackup(metadata, ignoredVdis)
Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
}
@@ -249,7 +48,8 @@ export class ImportVmBackup {
const vmRef = isFull
? await xapi.VM_import(backup, srRef)
: await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
newMacAddresses,
...this._importIncrementalVmSettings,
detectBase: false,
})
await Promise.all([
@@ -259,13 +59,6 @@ export class ImportVmBackup {
vmRef,
`${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
),
xapi.call(
'VM.set_name_description',
vmRef,
`Restored on ${formatFilenameDate(+new Date())} from ${adapter._handler._remote.name} -
${metadata.vm.name_description}
`
),
])
return {
@@ -276,5 +69,3 @@ export class ImportVmBackup {
)
}
}
decorateClass(ImportVmBackup, { _reuseNearestSnapshot: defer })

View File

@@ -35,8 +35,6 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -191,14 +189,13 @@ export class RemoteAdapter {
// check if we will be allowed to merge a a vhd created in this adapter
// with the vhd at path `path`
async isMergeableParent(packedParentUid, path) {
return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
return await Disposable.use(openVhd(this.handler, path), vhd => {
// this baseUuid is not linked with this vhd
if (!vhd.footer.uuid.equals(packedParentUid)) {
return false
}
// check if all the chain is composed of vhd directory
const isVhdDirectory = vhd.checkVhdsClass(VhdDirectory)
const isVhdDirectory = vhd instanceof VhdDirectory
return isVhdDirectory
? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
: !this.useVhdDirectory()
@@ -684,13 +681,11 @@ export class RemoteAdapter {
}
}
async outputStream(path, input, { checksum = true, maxStreamLength, streamLength, validator = noop } = {}) {
async outputStream(path, input, { checksum = true, validator = noop } = {}) {
const container = watchStreamSize(input)
await this._handler.outputStream(path, input, {
checksum,
dirMode: this._dirMode,
maxStreamLength,
streamLength,
async validator() {
await input.task
return validator.apply(this, arguments)
@@ -752,37 +747,10 @@ export class RemoteAdapter {
}
async readVmBackupMetadata(path) {
let json
let isImmutable = false
let remoteIsImmutable = false
// if the remote is immutable, check if this metadatas are also immutables
try {
// this file is not encrypted
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
remoteIsImmutable = true
} catch (error) {
if (error.code !== 'ENOENT') {
throw error
}
}
try {
// this will trigger an EPERM error if the file is immutable
json = await this.handler.readFile(path, { flag: 'r+' })
// s3 handler don't respect flags
} catch (err) {
// retry without triggerring immutbaility check ,only on immutable remote
if (err.code === 'EPERM' && remoteIsImmutable) {
isImmutable = true
json = await this._handler.readFile(path, { flag: 'r' })
} else {
throw err
}
}
// _filename is a private field used to compute the backup id
//
// it's enumerable to make it cacheable
const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
// backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
if (typeof metadata.vm.is_a_template === 'number') {

View File

@@ -21,7 +21,7 @@ export class RestoreMetadataBackup {
})
} else {
const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
const dataFileName = resolve(backupId, metadata.data ?? 'data.json')
const data = await handler.readFile(dataFileName)
// if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it

View File

@@ -2,10 +2,7 @@ import { createLogger } from '@xen-orchestra/log'
import { catchGlobalErrors } from '@xen-orchestra/log/configure'
import Disposable from 'promise-toolbox/Disposable'
import humanFormat from 'human-format'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import mapValues from 'lodash/mapValues.js'
import ms from 'ms'
import { compose } from '@vates/compose'
import { createCachedLookup } from '@vates/cached-dns.lookup'
import { createDebounceResource } from '@vates/disposable/debounceResource.js'
@@ -23,7 +20,7 @@ createCachedLookup().patchGlobal()
const logger = createLogger('xo:backups:worker')
catchGlobalErrors(logger)
const { debug, info } = logger
const { debug } = logger
class BackupWorker {
#config
@@ -152,27 +149,6 @@ process.on('message', async message => {
debug('message received', { message })
if (message.action === 'run') {
const resourceStart = process.resourceUsage()
const timeStart = process.hrtime.bigint()
info('starting backup')
process.on('exit', exitCode => {
const resourceUsage = mapValues(process.resourceUsage(), (end, key) => end - resourceStart[key])
const cpuTotal = resourceUsage.userCPUTime + resourceUsage.systemCPUTime
const duration = Number((process.hrtime.bigint() - timeStart) / 1000n) // in μs
info('process will exit', {
duration,
exitCode,
resourceUsage,
summary: {
duration: ms(duration / 1000),
cpuUsage: Math.round((100 * cpuTotal) / duration) + '%',
memoryUsage: humanFormat.bytes(resourceUsage.maxRSS * 1024),
},
})
})
const backupWorker = new BackupWorker(message.data)
try {
const result = message.runWithLogs
@@ -201,7 +177,6 @@ process.on('message', async message => {
status: 'failure',
})
} finally {
info('backup has ended')
await ignoreErrors.call(backupWorker.debounceResource.flushAll())
process.disconnect()
}

View File

@@ -67,11 +67,6 @@ async function generateVhd(path, opts = {}) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
if (opts.blocks) {
for (const blockId of opts.blocks) {
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
}
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
@@ -235,7 +230,7 @@ test('it merges delta of non destroyed chain', async () => {
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children after the merge
assert.equal(metadata.size, 104960)
assert.equal(metadata.size, 209920)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
@@ -325,7 +320,6 @@ describe('tests multiple combination ', () => {
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
blocks: [1, 3],
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
@@ -334,7 +328,6 @@ describe('tests multiple combination ', () => {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: ancestor.footer.uuid,
},
blocks: [1, 2],
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
@@ -344,7 +337,6 @@ describe('tests multiple combination ', () => {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: child.footer.uuid,
},
blocks: [2, 3],
})
// an older parent that was merging in clean
@@ -403,7 +395,7 @@ describe('tests multiple combination ', () => {
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
assert.deepEqual(metadata.size, vhdMode === 'file' ? 314880 : undefined)
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
// ancestor and child should be merged

View File

@@ -36,32 +36,34 @@ const computeVhdsSize = (handler, vhdPaths) =>
)
// chain is [ ancestor, child_1, ..., child_n ]
async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
logInfo(`merging VHD chain`, { chain })
async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
if (merge) {
logInfo(`merging VHD chain`, { chain })
let done, total
const handle = setInterval(() => {
if (done !== undefined) {
logInfo('merge in progress', {
done,
parent: chain[0],
progress: Math.round((100 * done) / total),
total,
let done, total
const handle = setInterval(() => {
if (done !== undefined) {
logInfo('merge in progress', {
done,
parent: chain[0],
progress: Math.round((100 * done) / total),
total,
})
}
}, 10e3)
try {
return await mergeVhdChain(handler, chain, {
logInfo,
mergeBlockConcurrency,
onProgress({ done: d, total: t }) {
done = d
total = t
},
removeUnused: remove,
})
} finally {
clearInterval(handle)
}
}, 10e3)
try {
return await mergeVhdChain(handler, chain, {
logInfo,
mergeBlockConcurrency,
onProgress({ done: d, total: t }) {
done = d
total = t
},
removeUnused: remove,
})
} finally {
clearInterval(handle)
}
}
@@ -437,8 +439,7 @@ export async function cleanVm(
}
}
// no warning because a VHD can be unused for perfectly good reasons,
// e.g. the corresponding backup (metadata file) has been deleted
logWarn('unused VHD', { path: vhd })
if (remove) {
logInfo('deleting unused VHD', { path: vhd })
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
@@ -470,20 +471,23 @@ export async function cleanVm(
const metadataWithMergedVhd = {}
const doMerge = async () => {
await asyncMap(toMerge, async chain => {
const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
const merged = await limitedMergeVhdChain(handler, chain, {
logInfo,
logWarn,
remove,
merge,
mergeBlockConcurrency,
})
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
if (merged !== undefined) {
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
metadataWithMergedVhd[metadataPath] = true
}
})
}
await Promise.all([
...unusedVhdsDeletion,
toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : () => Promise.resolve()),
toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
asyncMap(unusedXvas, path => {
logWarn('unused XVA', { path })
if (remove) {
@@ -505,11 +509,12 @@ export async function cleanVm(
// update size for delta metadata with merged VHD
// check for the other that the size is the same as the real file size
await asyncMap(jsons, async metadataPath => {
const metadata = backups.get(metadataPath)
let fileSystemSize
const mergedSize = metadataWithMergedVhd[metadataPath]
const merged = metadataWithMergedVhd[metadataPath] !== undefined
const { mode, size, vhds, xva } = metadata
@@ -519,29 +524,26 @@ export async function cleanVm(
const linkedXva = resolve('/', vmDir, xva)
try {
fileSystemSize = await handler.getSize(linkedXva)
if (fileSystemSize !== size && fileSystemSize !== undefined) {
logWarn('cleanVm: incorrect backup size in metadata', {
path: metadataPath,
actual: size ?? 'none',
expected: fileSystemSize,
})
}
} catch (error) {
// can fail with encrypted remote
}
} else if (mode === 'delta') {
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
// the size is not computed in some cases (e.g. VhdDirectory)
if (fileSystemSize === undefined) {
return
}
// don't warn if the size has changed after a merge
if (mergedSize === undefined) {
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
// the size is not computed in some cases (e.g. VhdDirectory)
if (fileSystemSize !== undefined && fileSystemSize !== size) {
logWarn('cleanVm: incorrect backup size in metadata', {
path: metadataPath,
actual: size ?? 'none',
expected: fileSystemSize,
})
}
if (!merged && fileSystemSize !== size) {
// FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
console.warn('cleanVm: incorrect backup size in metadata', {
path: metadataPath,
actual: size ?? 'none',
expected: fileSystemSize,
})
}
}
} catch (error) {
@@ -549,19 +551,9 @@ export async function cleanVm(
return
}
// systematically update size and differentials after a merge
// @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
metadata.size = mergedSize ?? fileSystemSize ?? size
if (mergedSize) {
// all disks are now key disk
metadata.isVhdDifferencing = {}
for (const id of Object.values(metadata.vdis ?? {})) {
metadata.isVhdDifferencing[`${id}.vhd`] = false
}
}
// systematically update size after a merge
if ((merged || fixMetadata) && size !== fileSystemSize) {
metadata.size = fileSystemSize
mustRegenerateCache = true
try {
await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })

View File

@@ -1,3 +1,4 @@
import find from 'lodash/find.js'
import groupBy from 'lodash/groupBy.js'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import omit from 'lodash/omit.js'
@@ -11,18 +12,24 @@ import { cancelableMap } from './_cancelableMap.mjs'
import { Task } from './Task.mjs'
import pick from 'lodash/pick.js'
// in `other_config` of an incrementally replicated VM, contains the UUID of the source VM
export const TAG_BASE_DELTA = 'xo:base_delta'
// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
//
// added after the complete replication
export const TAG_BACKUP_SR = 'xo:backup:sr'
// in other_config of VDIs of an incrementally replicated VM, contains the UUID of the source VDI
export const TAG_COPY_SRC = 'xo:copy_of'
const TAG_BACKUP_SR = 'xo:backup:sr'
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
const resolveUuid = async (xapi, cache, uuid, type) => {
if (uuid == null) {
return uuid
}
let ref = cache.get(uuid)
if (ref === undefined) {
ref = await xapi.call(`${type}.get_by_uuid`, uuid)
cache.set(uuid, ref)
}
return ref
}
export async function exportIncrementalVm(
vm,
@@ -34,8 +41,6 @@ export async function exportIncrementalVm(
fullVdisRequired = new Set(),
disableBaseTags = false,
nbdConcurrency = 1,
preferNbd,
} = {}
) {
// refs of VM's VDIs → base's VDIs.
@@ -83,8 +88,6 @@ export async function exportIncrementalVm(
baseRef: baseVdi?.$ref,
cancelToken,
format: 'vhd',
nbdConcurrency,
preferNbd,
})
})
@@ -142,7 +145,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
$defer,
incrementalVm,
sr,
{ cancelToken = CancelToken.none, newMacAddresses = false } = {}
{ cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
) {
const { version } = incrementalVm
if (compareVersions(version, '1.0.0') < 0) {
@@ -152,6 +155,35 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
const vmRecord = incrementalVm.vm
const xapi = sr.$xapi
let baseVm
if (detectBase) {
const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
if (remoteBaseVmUuid) {
baseVm = find(
xapi.objects.all,
obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
)
if (!baseVm) {
throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
}
}
}
const cache = new Map()
const mapVdisSrRefs = {}
for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
}
const baseVdis = {}
baseVm &&
baseVm.$VBDs.forEach(vbd => {
const vdi = vbd.$VDI
if (vdi !== undefined) {
baseVdis[vbd.VDI] = vbd.$VDI
}
})
const vdiRecords = incrementalVm.vdis
// 0. Create suspend_VDI
@@ -163,7 +195,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
})
} else {
suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
suspendVdi = await xapi.getRecord(
'VDI',
await xapi.VDI_create({
...vdi,
other_config: {
...vdi.other_config,
[TAG_BASE_DELTA]: undefined,
[TAG_COPY_SRC]: vdi.uuid,
},
sr: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
})
)
$defer.onFailure(() => suspendVdi.$destroy())
}
}
@@ -181,6 +224,10 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
ha_always_run: false,
is_a_template: false,
name_label: '[Importing…] ' + vmRecord.name_label,
other_config: {
...vmRecord.other_config,
[TAG_COPY_SRC]: vmRecord.uuid,
},
},
{
bios_strings: vmRecord.bios_strings,
@@ -201,8 +248,14 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
const vdi = vdiRecords[vdiRef]
let newVdi
if (vdi.baseVdi !== undefined) {
newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
if (remoteBaseVdiUuid) {
const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
if (!baseVdi) {
throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
}
newVdi = await xapi.getRecord('VDI', await baseVdi.$clone())
$defer.onFailure(() => newVdi.$destroy())
await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
@@ -213,7 +266,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
// suspendVDI has already created
newVdi = suspendVdi
} else {
newVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
newVdi = await xapi.getRecord(
'VDI',
await xapi.VDI_create({
...vdi,
other_config: {
...vdi.other_config,
[TAG_BASE_DELTA]: undefined,
[TAG_COPY_SRC]: vdi.uuid,
},
SR: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
})
)
$defer.onFailure(() => newVdi.$destroy())
}
@@ -252,19 +316,13 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
// Import VDI contents.
cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
for (let stream of ensureArray(streams[`${id}.vhd`])) {
if (stream === null) {
// we restore a backup and reuse completly a local snapshot
continue
}
if (typeof stream === 'function') {
stream = await stream()
}
if (stream.length === undefined) {
stream = await createVhdStreamWithLength(stream)
}
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
}
}),

View File

@@ -6,12 +6,11 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { runTask } from './_runTask.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
const noop = Function.prototype
const DEFAULT_REMOTE_VM_SETTINGS = {
concurrency: 2,
copyRetention: 0,
@@ -19,10 +18,8 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
exportRetention: 0,
healthCheckSr: undefined,
healthCheckVmsWithTags: [],
healthCheckTimeout: '10m',
maxExportRate: 0,
maxMergedDeltasPerRun: Infinity,
nRetriesVmBackupFailures: 0,
timeout: 0,
validateVhdStreams: false,
vmTimeout: 0,
@@ -44,7 +41,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
await Disposable.use(
() => this._getAdapter(job.sourceRemote),
() => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -66,19 +62,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmsUuids)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = vmUuid => {
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
nTriesByVmId[vmUuid]++
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
const opts = {
baseSettings,
@@ -87,7 +72,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
healthCheckSr,
remoteAdapters,
schedule,
settings: vmSettings,
settings: { ...settings, ...allSettings[vmUuid] },
sourceRemoteAdapter,
throttleStream,
vmUuid,
@@ -101,46 +86,10 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
}
return sourceRemoteAdapter
.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
.then(vmBackups => {
// avoiding to create tasks for empty directories
if (vmBackups.length > 0) {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
return task
.run(async () => {
try {
const result = await vmBackup.run()
task.success(result)
return result
} catch (error) {
if (isLastRun) {
throw error
} else {
Task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: error.message,
})
queue.add(vmUuid)
}
}
})
.catch(noop)
}
})
return runTask(taskStart, () => vmBackup.run())
}
const { concurrency } = settings
const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
while (queue.size > 0) {
const vmIds = Array.from(queue)
queue.clear()
await asyncMapSettled(vmIds, _handleVm)
}
await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
}
)
}

View File

@@ -11,8 +11,6 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
import { FullXapi } from './_vmRunners/FullXapi.mjs'
const noop = Function.prototype
const DEFAULT_XAPI_VM_SETTINGS = {
bypassVdiChainsCheck: false,
checkpointSnapshot: false,
@@ -24,10 +22,8 @@ const DEFAULT_XAPI_VM_SETTINGS = {
fullInterval: 0,
healthCheckSr: undefined,
healthCheckVmsWithTags: [],
healthCheckTimeout: '10m',
maxExportRate: 0,
maxMergedDeltasPerRun: Infinity,
nRetriesVmBackupFailures: 0,
offlineBackup: false,
offlineSnapshot: false,
snapshotRetention: 0,
@@ -57,7 +53,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
await Disposable.use(
Disposable.all(
extractIdsFromSimplePattern(job.srs).map(id =>
@@ -94,98 +89,48 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmIds)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = vmUuid => {
const getVmTask = () => {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
return taskByVmId[vmUuid]
}
const vmBackupFailed = error => {
if (isLastRun) {
throw error
} else {
Task.warning(`Retry the VM backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: error.message,
})
queue.add(vmUuid)
}
}
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
nTriesByVmId[vmUuid]++
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
return this._getRecord('VM', vmUuid).then(
disposableVm =>
Disposable.use(disposableVm, async vm => {
if (taskStart.data.name_label === undefined) {
taskStart.data.name_label = vm.name_label
}
const task = getVmTask()
return task
.run(async () => {
const opts = {
baseSettings,
config,
getSnapshotNameLabel,
healthCheckSr,
job,
remoteAdapters,
schedule,
settings: vmSettings,
srs,
throttleStream,
vm,
}
let vmBackup
if (job.mode === 'delta') {
vmBackup = new IncrementalXapi(opts)
Disposable.use(disposableVm, vm => {
taskStart.data.name_label = vm.name_label
return runTask(taskStart, () => {
const opts = {
baseSettings,
config,
getSnapshotNameLabel,
healthCheckSr,
job,
remoteAdapters,
schedule,
settings: { ...settings, ...allSettings[vm.uuid] },
srs,
throttleStream,
vm,
}
let vmBackup
if (job.mode === 'delta') {
vmBackup = new IncrementalXapi(opts)
} else {
if (job.mode === 'full') {
vmBackup = new FullXapi(opts)
} else {
if (job.mode === 'full') {
vmBackup = new FullXapi(opts)
} else {
throw new Error(`Job mode ${job.mode} not implemented`)
}
throw new Error(`Job mode ${job.mode} not implemented`)
}
try {
const result = await vmBackup.run()
task.success(result)
return result
} catch (error) {
vmBackupFailed(error)
}
})
.catch(noop) // errors are handled by logs
}
return vmBackup.run()
})
}),
error =>
getVmTask().run(() => {
vmBackupFailed(error)
runTask(taskStart, () => {
throw error
})
)
}
const { concurrency } = settings
const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
while (queue.size > 0) {
const vmIds = Array.from(queue)
queue.clear()
await asyncMapSettled(vmIds, _handleVm)
}
await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
}
)
}

View File

@@ -31,7 +31,7 @@ export class PoolMetadataBackup {
const poolDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${schedule.id}/${pool.$id}`
const dir = `${poolDir}/${formatFilenameDate(timestamp)}`
const stream = (await this._exportPoolMetadata()).body
const stream = await this._exportPoolMetadata()
const fileName = `${dir}/data`
const metadata = JSON.stringify(

View File

@@ -1,3 +1,5 @@
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { AbstractRemote } from './_AbstractRemote.mjs'
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
@@ -8,9 +10,15 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
_getRemoteWriter() {
return FullRemoteWriter
}
async _run() {
async _run($defer) {
const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
if (transferList.length > 0) {
for (const metadata of transferList) {
const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
@@ -21,8 +29,6 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
writer =>
writer.run({
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
streamLength: stream.length,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
@@ -38,3 +44,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
}
}
}
decorateMethodsWith(FullRemote, {
_run: defer,
})

View File

@@ -30,32 +30,18 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
const vm = this._vm
const exportedVm = this._exportedVm
const stream = this._throttleStream(
(
await this._xapi.VM_export(exportedVm.$ref, {
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
useSnapshot: false,
})
).body
await this._xapi.VM_export(exportedVm.$ref, {
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
useSnapshot: false,
})
)
const vdis = await exportedVm.$getDisks()
let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe
for (const vdiRef of vdis) {
const vdi = await this._xapi.getRecord('VDI', vdiRef)
// the size a of fully allocated vdi will be virtual_size exaclty, it's a gross over evaluation
// of the real stream size in general, since a disk is never completly full
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
maxStreamLength += vdi.virtual_size
}
const sizeContainer = watchStreamSize(stream)
const timestamp = Date.now()
await this._callWriters(
writer =>
writer.run({
maxStreamLength,
sizeContainer,
stream: forkStreamUnpipe(stream),
timestamp,

View File

@@ -1,6 +1,7 @@
import { asyncEach } from '@vates/async-each'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import assert from 'node:assert'
import * as UUID from 'uuid'
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
import mapValues from 'lodash/mapValues.js'
@@ -8,64 +9,33 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
import { forkDeltaExport } from './_forkDeltaExport.mjs'
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
import { Task } from '../../Task.mjs'
import { Disposable } from 'promise-toolbox'
import { openVhd } from 'vhd-lib'
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
_getRemoteWriter() {
return IncrementalRemoteWriter
}
async _selectBaseVm(metadata) {
// for each disk , get the parent
const baseUuidToSrcVdi = new Map()
// no previous backup for a base( =key) backup
if (metadata.isBase) {
return
}
await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
if (isDifferencing) {
const vmDir = getVmBackupDir(metadata.vm.uuid)
const path = `${vmDir}/${metadata.vhds[id]}`
// don't catch error : we can't recover if the source vhd are missing
await Disposable.use(openVhd(this._sourceRemoteAdapter._handler, path), vhd => {
baseUuidToSrcVdi.set(UUID.stringify(vhd.header.parentUuid), vdi.$snapshot_of$uuid)
})
}
})
const presentBaseVdis = new Map(baseUuidToSrcVdi)
await this._callWriters(
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
'writer.checkBaseVdis()',
false
)
// check if the parent vdi are present in all the remotes
baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
if (!presentBaseVdis.has(baseUuid)) {
throw new Error(`Missing vdi ${baseUuid} which is a base for a delta`)
}
})
// yeah , let's go
}
async _run() {
async _run($defer) {
const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
if (transferList.length > 0) {
for (const metadata of transferList) {
assert.strictEqual(metadata.mode, 'delta')
await this._selectBaseVm(metadata)
await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
useChain: false,
})
const isVhdDifferencing = {}
const differentialVhds = {}
await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
differentialVhds[key] = await isVhdDifferencingDisk(stream)
})
incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
@@ -73,24 +43,13 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
writer =>
writer.transfer({
deltaExport: forkDeltaExport(incrementalExport),
isVhdDifferencing,
differentialVhds,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
}),
'writer.transfer()'
)
// this will update parent name with the needed alias
await this._callWriters(
writer =>
writer.updateUuidAndChain({
isVhdDifferencing,
timestamp: metadata.timestamp,
vdis: incrementalExport.vdis,
}),
'writer.updateUuidAndChain()'
)
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
// for healthcheck
this._tags = metadata.vm.tags
@@ -102,3 +61,6 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
}
export const IncrementalRemote = IncrementalRemoteVmBackupRunner
decorateMethodsWith(IncrementalRemoteVmBackupRunner, {
_run: defer,
})

View File

@@ -41,8 +41,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
fullVdisRequired,
nbdConcurrency: this._settings.nbdConcurrency,
preferNbd: this._settings.preferNbd,
})
// since NBD is network based, if one disk use nbd , all the disk use them
// except the suspended VDI
@@ -50,11 +48,11 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
Task.info('Transfer data using NBD')
}
const isVhdDifferencing = {}
const differentialVhds = {}
// since isVhdDifferencingDisk is reading and unshifting data in stream
// it should be done BEFORE any other stream transform
await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
differentialVhds[key] = await isVhdDifferencingDisk(stream)
})
const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
@@ -69,7 +67,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
writer =>
writer.transfer({
deltaExport: forkDeltaExport(deltaExport),
isVhdDifferencing,
differentialVhds,
sizeContainers,
timestamp,
vm,
@@ -78,18 +76,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
'writer.transfer()'
)
// we want to control the uuid of the vhd in the chain
// and ensure they are correctly chained
await this._callWriters(
writer =>
writer.updateUuidAndChain({
isVhdDifferencing,
timestamp,
vdis: deltaExport.vdis,
}),
'writer.updateUuidAndChain()'
)
this._baseVm = exportedVm
if (baseVm !== undefined) {
@@ -145,7 +131,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
])
const srcVdi = srcVdis[snapshotOf]
if (srcVdi !== undefined) {
baseUuidToSrcVdi.set(baseUuid, srcVdi.uuid)
baseUuidToSrcVdi.set(baseUuid, srcVdi)
} else {
debug('ignore snapshot VDI because no longer present on VM', {
vdi: baseUuid,
@@ -166,18 +152,18 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
}
const fullVdisRequired = new Set()
baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
if (presentBaseVdis.has(baseUuid)) {
debug('found base VDI', {
base: baseUuid,
vdi: srcVdiUuid,
vdi: srcVdi.uuid,
})
} else {
debug('missing base VDI', {
base: baseUuid,
vdi: srcVdiUuid,
vdi: srcVdi.uuid,
})
fullVdisRequired.add(srcVdiUuid)
fullVdisRequired.add(srcVdi.uuid)
}
})

View File

@@ -1,12 +1,9 @@
import { asyncEach } from '@vates/async-each'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { Disposable } from 'promise-toolbox'
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
import { Abstract } from './_Abstract.mjs'
import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
constructor({
@@ -24,7 +21,6 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this.config = config
this.job = job
this.remoteAdapters = remoteAdapters
this._settings = settings
this.scheduleId = schedule.id
this.timestamp = undefined
@@ -38,8 +34,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this._writers = writers
const RemoteWriter = this._getRemoteWriter()
extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
const adapter = remoteAdapters[remoteId]
Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
const targetSettings = {
...settings,
...allSettings[remoteId],
@@ -92,21 +87,11 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
return chain
}
async run($defer) {
async run() {
const handler = this._sourceRemoteAdapter._handler
await Disposable.use(await handler.lock(getVmBackupDir(this._vmUuid)), async () => {
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
await this._run()
await this._healthCheck()
})
}
}
decorateMethodsWith(AbstractRemote, {
run: defer,
})

View File

@@ -31,11 +31,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
throw new Error('cannot backup a VM created by this very job')
}
const currentOperations = Object.values(vm.current_operations)
if (currentOperations.some(_ => _ === 'migrate_send' || _ === 'pool_migrate')) {
throw new Error('cannot backup a VM currently being migrated')
}
this.config = config
this.job = job
this.remoteAdapters = remoteAdapters
@@ -240,18 +235,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
})
}, 'writer.beforeBackup()')
const vm = this._vm
// block migration during the backup on the VM itself, not the latest snapshot
{
const { pool_migrate = null, migrate_send = null } = vm.blocked_operations
const reason = 'VM migration is blocked during backup'
await vm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
$defer(() => vm.update_blocked_operations({ pool_migrate, migrate_send }))
}
await this._fetchJobSnapshots()
await this._selectBaseVm()
@@ -259,6 +242,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await this._cleanMetadata()
await this._removeUnusedSnapshots()
const vm = this._vm
const isRunning = vm.power_state === 'Running'
const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
if (startAfter) {

View File

@@ -1,11 +1,11 @@
import cloneDeep from 'lodash/cloneDeep.js'
import mapValues from 'lodash/mapValues.js'
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
export function forkDeltaExport(deltaExport) {
const { streams, ...rest } = deltaExport
const newMetadata = cloneDeep(rest)
newMetadata.streams = mapValues(streams, forkStreamUnpipe)
return newMetadata
return Object.create(deltaExport, {
streams: {
value: mapValues(deltaExport.streams, forkStreamUnpipe),
},
})
}

View File

@@ -24,7 +24,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
)
}
async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
async _run({ timestamp, sizeContainer, stream, vm, vmSnapshot }) {
const settings = this._settings
const job = this._job
const scheduleId = this._scheduleId
@@ -65,8 +65,6 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
await Task.run({ name: 'transfer' }, async () => {
await adapter.outputStream(dataFilename, stream, {
maxStreamLength,
streamLength,
validator: tmpPath => adapter.isValidXva(tmpPath),
})
return { size: sizeContainer.size }

View File

@@ -1,12 +1,13 @@
import assert from 'node:assert'
import mapValues from 'lodash/mapValues.js'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { asyncEach } from '@vates/async-each'
import { asyncMap } from '@xen-orchestra/async-map'
import { chainVhd, openVhd } from 'vhd-lib'
import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
import { createLogger } from '@xen-orchestra/log'
import { decorateClass } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { dirname, basename } from 'node:path'
import { dirname } from 'node:path'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -21,45 +22,42 @@ import { Disposable } from 'promise-toolbox'
const { warn } = createLogger('xo:backups:DeltaBackupWriter')
export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
#parentVdiPaths
#vhds
async checkBaseVdis(baseUuidToSrcVdi) {
this.#parentVdiPaths = {}
const { handler } = this._adapter
const adapter = this._adapter
const vdisDir = `${this._vmBackupDir}/vdis/${this._job.id}`
await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdiUuid]) => {
let parentDestPath
const vhdDir = `${vdisDir}/${srcVdiUuid}`
await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
let found = false
try {
const vhds = await handler.list(vhdDir, {
const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
ignoreMissing: true,
prependDir: true,
})
const packedBaseUuid = packUuid(baseUuid)
// the last one is probably the right one
for (let i = vhds.length - 1; i >= 0 && parentDestPath === undefined; i--) {
const path = vhds[i]
await asyncMap(vhds, async path => {
try {
if (await adapter.isMergeableParent(packedBaseUuid, path)) {
parentDestPath = path
}
await checkVhdChain(handler, path)
// Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
//
// since all the checks of a path are done in parallel, found would be containing
// only the last answer of isMergeableParent which is probably not the right one
// this led to the support tickets https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
found = found || isMergeable
} catch (error) {
warn('checkBaseVdis', { error })
await ignoreErrors.call(VhdAbstract.unlink(handler, path))
}
}
})
} catch (error) {
warn('checkBaseVdis', { error })
}
// no usable parent => the runner will have to decide to fall back to a full or stop backup
if (parentDestPath === undefined) {
if (!found) {
baseUuidToSrcVdi.delete(baseUuid)
} else {
this.#parentVdiPaths[vhdDir] = parentDestPath
}
})
}
@@ -124,44 +122,6 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
}
}
async updateUuidAndChain({ isVhdDifferencing, vdis }) {
assert.notStrictEqual(
this.#vhds,
undefined,
'_transfer must be called before updateUuidAndChain for incremental backups'
)
const parentVdiPaths = this.#parentVdiPaths
const { handler } = this._adapter
const vhds = this.#vhds
await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
const isDifferencing = isVhdDifferencing[`${id}.vhd`]
const path = `${this._vmBackupDir}/${vhds[id]}`
if (isDifferencing) {
assert.notStrictEqual(
parentVdiPaths,
'checkbasevdi must be called before updateUuidAndChain for incremental backups'
)
const parentPath = parentVdiPaths[dirname(path)]
// we are in a incremental backup
// we already computed the chain in checkBaseVdis
assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
// forbid any kind of loop
assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
await chainVhd(handler, parentPath, handler, path)
}
// set the correct UUID in the VHD if needed
await Disposable.use(openVhd(handler, path), async vhd => {
if (!vhd.footer.uuid.equals(packUuid(vdi.uuid))) {
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
}
})
})
}
async _deleteOldEntries() {
const adapter = this._adapter
const oldEntries = this._oldEntries
@@ -172,7 +132,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
}
}
async _transfer($defer, { isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
async _transfer($defer, { differentialVhds, timestamp, deltaExport, vm, vmSnapshot }) {
const adapter = this._adapter
const job = this._job
const scheduleId = this._scheduleId
@@ -180,10 +140,14 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
const jobId = job.id
const handler = adapter.handler
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// @todo : should skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
}
const basename = formatFilenameDate(timestamp)
// update this.#vhds before eventually skipping transfer, so that
// updateUuidAndChain has all the mandatory data
const vhds = (this.#vhds = mapValues(
const vhds = mapValues(
deltaExport.vdis,
vdi =>
`vdis/${jobId}/${
@@ -193,18 +157,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
vdi.uuid
: vdi.$snapshot_of$uuid
}/${adapter.getVhdFileName(basename)}`
))
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
Object.values(deltaExport.streams).forEach(stream => stream.destroy())
return { size: 0 }
}
)
metadataContent = {
isVhdDifferencing,
jobId,
mode: job.mode,
scheduleId,
@@ -217,24 +172,56 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
vm,
vmSnapshot,
}
const { size } = await Task.run({ name: 'transfer' }, async () => {
let transferSize = 0
await asyncEach(
Object.keys(deltaExport.vdis),
async id => {
Object.entries(deltaExport.vdis),
async ([id, vdi]) => {
const path = `${this._vmBackupDir}/${vhds[id]}`
// don't write it as transferSize += await async function
// since i += await asyncFun lead to race condition
// as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
const isDelta = differentialVhds[`${id}.vhd`]
let parentPath
if (isDelta) {
const vdiDir = dirname(path)
parentPath = (
await handler.list(vdiDir, {
filter: filename => filename[0] !== '.' && filename.endsWith('.vhd'),
prependDir: true,
})
)
.sort()
.pop()
assert.notStrictEqual(
parentPath,
undefined,
`missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config['xo:base_delta']}`
)
parentPath = parentPath.slice(1) // remove leading slash
// TODO remove when this has been done before the export
await checkVhd(handler, parentPath)
}
transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
// no checksum for VHDs, because they will be invalidated by
// merges and chainings
checksum: false,
validator: tmpPath => checkVhd(handler, tmpPath),
writeBlockConcurrency: this._config.writeBlockConcurrency,
})
transferSize += transferSizeOneDisk
if (isDelta) {
await chainVhd(handler, parentPath, handler, path)
}
// set the correct UUID in the VHD
await Disposable.use(openVhd(handler, path), async vhd => {
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
})
},
{
concurrency: settings.diskPerVmConcurrency,

View File

@@ -1,21 +1,18 @@
import assert from 'node:assert'
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { formatDateTime } from '@xen-orchestra/xapi'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
import { importIncrementalVm, TAG_BACKUP_SR, TAG_BASE_DELTA, TAG_COPY_SRC } from '../../_incrementalVm.mjs'
import { importIncrementalVm, TAG_COPY_SRC } from '../../_incrementalVm.mjs'
import { Task } from '../../Task.mjs'
import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
import { listReplicatedVms } from './_listReplicatedVms.mjs'
import find from 'lodash/find.js'
export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
assert.notStrictEqual(baseVm, undefined)
const sr = this._sr
const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(
vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
@@ -38,9 +35,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
}
}
}
updateUuidAndChain() {
// nothing to do, the chaining is not modified in this case
}
prepare({ isFull }) {
// create the task related to this export and ensure all methods are called in this context
const task = new Task({
@@ -86,54 +81,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
}
#decorateVmMetadata(backup) {
const { _warmMigration } = this._settings
const sr = this._sr
const xapi = sr.$xapi
const vm = backup.vm
vm.other_config[TAG_COPY_SRC] = vm.uuid
const remoteBaseVmUuid = vm.other_config[TAG_BASE_DELTA]
let baseVm
if (remoteBaseVmUuid) {
baseVm = find(
xapi.objects.all,
obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
)
if (!baseVm) {
throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
}
}
const baseVdis = {}
baseVm?.$VBDs.forEach(vbd => {
const vdi = vbd.$VDI
if (vdi !== undefined) {
baseVdis[vbd.VDI] = vbd.$VDI
}
})
vm.other_config[TAG_COPY_SRC] = vm.uuid
if (!_warmMigration) {
vm.tags.push('Continuous Replication')
}
Object.values(backup.vdis).forEach(vdi => {
vdi.other_config[TAG_COPY_SRC] = vdi.uuid
vdi.SR = sr.$ref
// vdi.other_config[TAG_BASE_DELTA] is never defined on a suspend vdi
if (vdi.other_config[TAG_BASE_DELTA]) {
const remoteBaseVdiUuid = vdi.other_config[TAG_BASE_DELTA]
const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
if (!baseVdi) {
throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
}
vdi.baseVdi = baseVdi
}
})
return backup
}
async _transfer({ timestamp, deltaExport, sizeContainers, vm }) {
const { _warmMigration } = this._settings
const sr = this._sr
@@ -144,7 +91,16 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
let targetVmRef
await Task.run({ name: 'transfer' }, async () => {
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
targetVmRef = await importIncrementalVm(
{
__proto__: deltaExport,
vm: {
...deltaExport.vm,
tags: _warmMigration ? deltaExport.vm.tags : [...deltaExport.vm.tags, 'Continuous Replication'],
},
},
sr
)
return {
size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
}
@@ -165,13 +121,13 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
)
),
targetVm.update_other_config({
[TAG_BACKUP_SR]: srUuid,
'xo:backup:sr': srUuid,
// these entries need to be added in case of offline backup
'xo:backup:datetime': formatDateTime(timestamp),
'xo:backup:job': job.id,
'xo:backup:schedule': scheduleId,
[TAG_BASE_DELTA]: vm.uuid,
'xo:backup:vm': vm.uuid,
}),
])
}

View File

@@ -1,9 +1,9 @@
import { AbstractWriter } from './_AbstractWriter.mjs'
export class AbstractFullWriter extends AbstractWriter {
async run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
async run({ timestamp, sizeContainer, stream, vm, vmSnapshot }) {
try {
return await this._run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot })
return await this._run({ timestamp, sizeContainer, stream, vm, vmSnapshot })
} finally {
// ensure stream is properly closed
stream.destroy()

View File

@@ -5,10 +5,6 @@ export class AbstractIncrementalWriter extends AbstractWriter {
throw new Error('Not implemented')
}
updateUuidAndChain() {
throw new Error('Not implemented')
}
cleanup() {
throw new Error('Not implemented')
}

View File

@@ -8,7 +8,6 @@ import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
import { ImportVmBackup } from '../../ImportVmBackup.mjs'
import { Task } from '../../Task.mjs'
import * as MergeWorker from '../../merge-worker/index.mjs'
import ms from 'ms'
const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
@@ -78,11 +77,11 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
healthCheck() {
const sr = this._healthCheckSr
assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
if (this._metadataFileName === undefined) {
// this can happen when making a mirror backup with nothing to transfer
Task.info('no health check, since no backup have been transferred')
return
}
assert.notStrictEqual(
this._metadataFileName,
undefined,
'Metadata file name should be defined before making a health check'
)
return Task.run(
{
name: 'health check',
@@ -97,21 +96,11 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
metadata,
srUuid,
xapi,
settings: {
additionnalVmTag: 'xo:no-bak=Health Check',
},
}).run()
let restoredVm
const restoredVm = xapi.getObject(restoredId)
try {
restoredVm = xapi.getObject(restoredId)
} catch (err) {
restoredVm = await xapi.waitObject(restoredId)
}
try {
const timeout = ms(this._settings.healthCheckTimeout)
await new HealthCheckVmBackup({
restoredVm,
timeout,
xapi,
}).run()
} finally {
@@ -121,13 +110,13 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
)
}
async _isAlreadyTransferred(timestamp) {
_isAlreadyTransferred(timestamp) {
const vmUuid = this._vmUuid
const adapter = this._adapter
const backupDir = getVmBackupDir(vmUuid)
try {
const actualMetadata = JSON.parse(
await adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
)
return actualMetadata
} catch (error) {}

View File

@@ -3,7 +3,6 @@ import assert from 'node:assert/strict'
import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
import { Task } from '../../Task.mjs'
import ms from 'ms'
export const MixinXapiWriter = (BaseClass = Object) =>
class MixinXapiWriter extends BaseClass {
@@ -27,8 +26,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
}
healthCheck() {
// the SR that the VM has been replicated on
const sr = this._sr
const sr = this._healthCheckSr
assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
// copy VM
@@ -40,12 +38,8 @@ export const MixinXapiWriter = (BaseClass = Object) =>
const { $xapi: xapi } = sr
let healthCheckVmRef
try {
let baseVm
try {
baseVm = xapi.getObject(this._targetVmRef)
} catch (err) {
baseVm = await xapi.waitObject(this._targetVmRef)
}
const baseVm = xapi.getObject(this._targetVmRef) ?? (await xapi.waitObject(this._targetVmRef))
if (await this.#isAlreadyOnHealthCheckSr(baseVm)) {
healthCheckVmRef = await Task.run(
{ name: 'cloning-vm' },
@@ -59,22 +53,14 @@ export const MixinXapiWriter = (BaseClass = Object) =>
{ name: 'copying-vm' },
async () =>
await xapi
.callAsync(
'VM.copy',
this._targetVmRef,
`Health Check - ${baseVm.name_label}`,
this._healthCheckSr.$ref
)
.callAsync('VM.copy', this._targetVmRef, `Health Check - ${baseVm.name_label}`, sr.$ref)
.then(extractOpaqueRef)
)
}
const healthCheckVm =
xapi.getObject(healthCheckVmRef, undefined) ?? (await xapi.waitObject(healthCheckVmRef))
await healthCheckVm.add_tags('xo:no-bak=Health Check')
const timeout = ms(this._settings.healthCheckTimeout)
const healthCheckVm = xapi.getObject(healthCheckVmRef) ?? (await xapi.waitObject(healthCheckVmRef))
await new HealthCheckVmBackup({
restoredVm: healthCheckVm,
timeout,
xapi,
}).run()
} finally {

View File

@@ -221,7 +221,7 @@ For multiple objects:
### Settings
Settings are described in [`@xen-orchestra/backups/\_runners/VmsXapi.mjs``](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/_runners/VmsXapi.mjs).
Settings are described in [`@xen-orchestra/backups/Backup.js](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/Backup.js).
## Writer API
@@ -230,7 +230,6 @@ Settings are described in [`@xen-orchestra/backups/\_runners/VmsXapi.mjs``](http
- `checkBaseVdis(baseUuidToSrcVdi, baseVm)`
- `prepare({ isFull })`
- `transfer({ timestamp, deltaExport, sizeContainers })`
- `updateUuidAndChain({ isVhdDifferencing, vdis })`
- `cleanup()`
- `healthCheck()` // is not executed if no health check sr or tag doesn't match
- **Full**

View File

@@ -2,21 +2,6 @@ import mapValues from 'lodash/mapValues.js'
import { dirname } from 'node:path'
function formatVmBackup(backup) {
const { isVhdDifferencing, vmSnapshot } = backup
let differencingVhds
let dynamicVhds
// some backups don't use snapshots, therefore cannot be with memory
const withMemory = vmSnapshot !== undefined && vmSnapshot.suspend_VDI !== 'OpaqueRef:NULL'
// isVhdDifferencing is either undefined or an object
if (isVhdDifferencing !== undefined) {
differencingVhds = Object.values(isVhdDifferencing).filter(t => t).length
dynamicVhds = Object.values(isVhdDifferencing).filter(t => !t).length
if (withMemory) {
// the suspend VDI (memory) is always a dynamic
dynamicVhds -= 1
}
}
return {
disks:
backup.vhds === undefined
@@ -31,7 +16,6 @@ function formatVmBackup(backup) {
}),
id: backup.id,
isImmutable: backup.isImmutable,
jobId: backup.jobId,
mode: backup.mode,
scheduleId: backup.scheduleId,
@@ -41,10 +25,6 @@ function formatVmBackup(backup) {
name_description: backup.vm.name_description,
name_label: backup.vm.name_label,
},
differencingVhds,
dynamicVhds,
withMemory,
}
}

View File

@@ -2,21 +2,18 @@
// eslint-disable-next-line eslint-comments/disable-enable-pair
/* eslint-disable n/shebang */
import { asyncEach } from '@vates/async-each'
import { catchGlobalErrors } from '@xen-orchestra/log/configure'
import { createLogger } from '@xen-orchestra/log'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { join } from 'node:path'
import { load as loadConfig } from 'app-conf'
import Disposable from 'promise-toolbox/Disposable'
import min from 'lodash/min.js'
import { getVmBackupDir } from '../_getVmBackupDir.mjs'
import { RemoteAdapter } from '../RemoteAdapter.mjs'
import { CLEAN_VM_QUEUE } from './index.mjs'
const APP_NAME = 'xo-merge-worker'
const APP_DIR = new URL('.', import.meta.url).pathname
// -------------------------------------------------------------------
catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
@@ -37,7 +34,6 @@ const main = Disposable.wrap(async function* main(args) {
for (let i = 0; i < 10; ++i) {
const entries = await handler.list(CLEAN_VM_QUEUE)
if (entries.length !== 0) {
entries.sort()
return entries
}
await new Promise(timeoutResolver)
@@ -46,47 +42,38 @@ const main = Disposable.wrap(async function* main(args) {
let taskFiles
while ((taskFiles = await listRetry()) !== undefined) {
const { concurrency } = await loadConfig(APP_NAME, {
appDir: APP_DIR,
ignoreUnknownFormats: true,
})
await asyncEach(
taskFiles,
async taskFileBasename => {
const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
const taskFileBasename = min(taskFiles)
const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
// move this task to the end
try {
await handler.rename(previousTaskFile, taskFile)
} catch (error) {
// this error occurs if the task failed too many times (i.e. too many `_` prefixes)
// there is nothing more that can be done
if (error.code === 'ENAMETOOLONG') {
await handler.unlink(previousTaskFile)
}
// move this task to the end
try {
await handler.rename(previousTaskFile, taskFile)
} catch (error) {
// this error occurs if the task failed too many times (i.e. too many `_` prefixes)
// there is nothing more that can be done
if (error.code === 'ENAMETOOLONG') {
await handler.unlink(previousTaskFile)
}
throw error
}
try {
const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
try {
await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
} catch (error) {
// consider the clean successful if the VM dir is missing
if (error.code !== 'ENOENT') {
throw error
}
}
try {
const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
try {
await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
} catch (error) {
// consider the clean successful if the VM dir is missing
if (error.code !== 'ENOENT') {
throw error
}
}
handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
} catch (error) {
warn('failure handling task', { error })
}
},
{ concurrency }
)
handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
} catch (error) {
warn('failure handling task', { error })
}
}
})

View File

@@ -1 +0,0 @@
concurrency = 1

View File

@@ -8,7 +8,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.48.1",
"version": "0.40.0",
"engines": {
"node": ">=14.18"
},
@@ -17,48 +17,44 @@
"test-integration": "node--test *.integ.mjs"
},
"dependencies": {
"@iarna/toml": "^2.2.5",
"@kldzj/stream-throttle": "^1.1.1",
"@vates/async-each": "^1.0.0",
"@vates/cached-dns.lookup": "^1.0.0",
"@vates/compose": "^2.1.0",
"@vates/decorate-with": "^2.1.0",
"@vates/disposable": "^0.1.5",
"@vates/fuse-vhd": "^2.1.0",
"@vates/nbd-client": "^3.0.1",
"@vates/decorate-with": "^2.0.0",
"@vates/disposable": "^0.1.4",
"@vates/fuse-vhd": "^2.0.0",
"@vates/nbd-client": "^2.0.0",
"@vates/parse-duration": "^0.1.1",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/fs": "^4.1.6",
"@xen-orchestra/fs": "^4.0.1",
"@xen-orchestra/log": "^0.6.0",
"@xen-orchestra/template": "^0.1.0",
"app-conf": "^3.0.0",
"compare-versions": "^6.0.0",
"d3-time-format": "^4.1.0",
"decorator-synchronized": "^0.6.0",
"golike-defer": "^0.5.1",
"human-format": "^1.2.0",
"limit-concurrency-decorator": "^0.6.0",
"limit-concurrency-decorator": "^0.5.0",
"lodash": "^4.17.20",
"ms": "^2.1.3",
"node-zone": "^0.4.0",
"parse-pairs": "^2.0.0",
"promise-toolbox": "^0.21.0",
"proper-lockfile": "^4.1.2",
"tar": "^6.1.15",
"uuid": "^9.0.0",
"vhd-lib": "^4.9.2",
"xen-api": "^3.0.1",
"vhd-lib": "^4.5.0",
"xen-api": "^1.3.4",
"yazl": "^2.5.1"
},
"devDependencies": {
"fs-extra": "^11.1.0",
"rimraf": "^5.0.1",
"sinon": "^18.0.0",
"sinon": "^15.0.1",
"test": "^3.2.1",
"tmp": "^0.2.1"
},
"peerDependencies": {
"@xen-orchestra/xapi": "^5.0.1"
"@xen-orchestra/xapi": "^3.0.0"
},
"license": "AGPL-3.0-or-later",
"author": {

View File

@@ -1,10 +1,11 @@
#!/usr/bin/env node
import { defer } from 'golike-defer'
import { readFileSync } from 'fs'
import { Ref, Xapi } from 'xen-api'
'use strict'
const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url)))
const { Ref, Xapi } = require('xen-api')
const { defer } = require('golike-defer')
const pkg = require('./package.json')
Xapi.prototype.getVmDisks = async function (vm) {
const disks = { __proto__: null }

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/cr-seed-cli",
"version": "1.0.1",
"version": "0.2.0",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
@@ -10,15 +10,15 @@
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=10"
"node": ">=8"
},
"bin": {
"xo-cr-seed": "./index.mjs"
"xo-cr-seed": "./index.js"
},
"preferGlobal": true,
"dependencies": {
"golike-defer": "^0.5.1",
"xen-api": "^3.0.1"
"xen-api": "^1.3.4"
},
"scripts": {
"postversion": "npm publish"

View File

@@ -42,7 +42,7 @@
"test": "node--test"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^15.0.1",
"test": "^3.2.1"
}
}

Some files were not shown because too many files have changed in this diff Show More