Compare commits

..

1 Commits

Author SHA1 Message Date
Julien Fontanet
f8fac876ca WiP 2023-12-22 09:56:44 +01:00
1165 changed files with 24104 additions and 70316 deletions

View File

@@ -15,10 +15,9 @@ module.exports = {
overrides: [
{
files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js', '**/scripts/**.{,c,m}js'],
files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
rules: {
'n/no-process-exit': 'off',
'n/shebang': 'off',
'no-console': 'off',
},
},
@@ -39,12 +38,6 @@ module.exports = {
version: '>=16',
},
],
'n/no-unsupported-features/es-builtins': [
'error',
{
version: '>=16',
},
],
'n/no-unsupported-features/es-syntax': [
'error',
{
@@ -53,216 +46,6 @@ module.exports = {
],
},
},
{
files: ['@vates/**/*.{vue,ts}', '@xen-orchestra/{web-core,lite,web}/**/*.{vue,ts}'],
parser: 'vue-eslint-parser',
parserOptions: {
sourceType: 'module',
parser: '@typescript-eslint/parser',
},
plugins: ['import'],
extends: [
'plugin:import/recommended',
'plugin:import/typescript',
'plugin:@typescript-eslint/recommended',
'plugin:vue/vue3-essential',
],
settings: {
'import/resolver': {
typescript: true,
'eslint-import-resolver-custom-alias': {
alias: {
'@core': '../web-core/lib',
'@': './src',
},
extensions: ['.ts'],
packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
},
},
},
globals: {
XO_LITE_VERSION: true,
XO_LITE_GIT_HEAD: true,
},
rules: {
// allow "use strict"
strict: 'off',
// allow "require"
'@typescript-eslint/no-var-requires': 'off',
'import/order': [
'error',
{
groups: ['builtin', 'object', 'internal', ['type', 'external']],
alphabetize: {
order: 'asc',
caseInsensitive: true,
},
pathGroups: [
{
pattern: '@/**',
group: 'internal',
},
{
pattern: '@core/**',
group: 'internal',
},
],
},
],
'no-void': ['error', { allowAsStatement: true }],
'n/no-missing-import': 'off', // using 'import' plugin instead, to support TS aliases
'no-redeclare': 'off', // automatically checked by the TypeScript compiler
'no-dupe-class-members': 'off', // automatically checked by the TypeScript compiler
'no-undef': 'off', // automatically checked by the TypeScript compiler
'no-use-before-define': 'off', // replaced with the rule below
'@typescript-eslint/no-use-before-define': ['error', { functions: false }],
'@typescript-eslint/no-explicit-any': 'off',
'vue/multi-word-component-names': 'off',
// Vue 3 - Strongly Recommended
'vue/attribute-hyphenation': 'error',
'vue/component-definition-name-casing': 'error',
'vue/first-attribute-linebreak': 'error',
'vue/html-closing-bracket-newline': 'off', // Conflicts with Prettier
'vue/html-closing-bracket-spacing': 'error',
'vue/html-end-tags': 'error',
'vue/html-indent': 'off', // Conflicts with Prettier
'vue/html-quotes': 'error',
'vue/html-self-closing': ['error', { html: { void: 'always', normal: 'always', component: 'always' } }],
'vue/max-attributes-per-line': 'off', // Conflicts with Prettier
'vue/multiline-html-element-content-newline': 'error',
'vue/mustache-interpolation-spacing': 'error',
'vue/no-multi-spaces': 'error',
'vue/no-spaces-around-equal-signs-in-attribute': 'error',
'vue/no-template-shadow': 'error',
'vue/prop-name-casing': 'error',
'vue/require-default-prop': 'off', // https://github.com/vuejs/eslint-plugin-vue/issues/2051
'vue/require-explicit-emits': 'error',
'vue/require-prop-types': 'error',
'vue/singleline-html-element-content-newline': 'off',
'vue/v-bind-style': 'error',
'vue/v-on-event-hyphenation': ['error', 'always', { autofix: true }],
'vue/v-on-style': 'error',
'vue/v-slot-style': 'error',
// Vue 3 - Recommended
'vue/attributes-order': 'error',
'vue/no-lone-template': 'error',
'vue/no-multiple-slot-args': 'error',
'vue/no-v-html': 'error',
'vue/this-in-template': 'error',
// Vue 3 - Uncategorized
'vue/block-lang': ['error', { script: { lang: 'ts' }, style: { lang: 'postcss' } }],
'vue/block-order': ['error', { order: ['template', 'script', 'style'] }],
'vue/block-tag-newline': ['error', { singleline: 'consistent', multiline: 'always', maxEmptyLines: 0 }],
'vue/component-api-style': ['error', ['script-setup']],
'vue/component-name-in-template-casing': ['error', 'PascalCase', { registeredComponentsOnly: false }],
'vue/custom-event-name-casing': ['error', 'camelCase'],
'vue/define-emits-declaration': ['error', 'type-literal'],
'vue/define-macros-order': [
'error',
{
order: ['defineOptions', 'defineProps', 'defineEmits', 'defineModel', 'defineSlots'],
defineExposeLast: true,
},
],
'vue/define-props-declaration': ['error', 'type-based'],
'vue/enforce-style-attribute': ['error', { allow: ['scoped', 'module'] }],
'vue/html-button-has-type': 'error',
'vue/html-comment-content-newline': ['error', { singleline: 'never', multiline: 'always' }],
'vue/html-comment-content-spacing': ['error', 'always'],
'vue/no-duplicate-attr-inheritance': 'error',
'vue/no-empty-component-block': 'error',
'vue/no-multiple-objects-in-class': 'error',
'vue/no-ref-object-reactivity-loss': 'error',
'vue/no-required-prop-with-default': 'error',
'vue/no-static-inline-styles': 'error',
'vue/no-template-target-blank': 'error',
'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
'vue/no-undef-properties': 'error',
'vue/no-unused-refs': 'error',
'vue/no-use-v-else-with-v-for': 'error',
'vue/no-useless-mustaches': 'error',
'vue/no-useless-v-bind': 'error',
'vue/no-v-text': 'error',
'vue/padding-line-between-blocks': 'error',
'vue/prefer-define-options': 'error',
'vue/prefer-separate-static-class': 'error',
'vue/prefer-true-attribute-shorthand': 'error',
'vue/require-macro-variable-name': 'error',
'vue/valid-define-options': 'error',
},
},
// Specific rules for XO i18n-related files
{
files: [
'@xen-orchestra/lite/**/*.{vue,ts,json}',
'@xen-orchestra/web/**/*.{vue,ts,json}',
'@xen-orchestra/web-core/**/*.{vue,ts,json}',
],
extends: ['plugin:@intlify/vue-i18n/base'],
settings: {
'vue-i18n': {
localeDir: [
'@xen-orchestra/lite/src/locales/*.json',
'@xen-orchestra/web/src/locales/*.json',
'@xen-orchestra/web-core/lib/locales/*.json',
],
messageSyntaxVersion: '^9.9.0',
},
},
rules: {
strict: 'off',
'no-irregular-whitespace': 'off',
'@intlify/vue-i18n/no-deprecated-i18n-component': 'error',
'@intlify/vue-i18n/no-deprecated-i18n-place-attr': 'error',
'@intlify/vue-i18n/no-deprecated-i18n-places-prop': 'error',
'@intlify/vue-i18n/no-deprecated-modulo-syntax': 'error',
'@intlify/vue-i18n/no-deprecated-tc': 'error',
'@intlify/vue-i18n/no-html-messages': 'error',
'@intlify/vue-i18n/no-i18n-t-path-prop': 'error',
'@intlify/vue-i18n/no-missing-keys': 'error',
'@intlify/vue-i18n/no-raw-text': [
'error',
{
ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
},
],
'@intlify/vue-i18n/no-v-html': 'error',
'@intlify/vue-i18n/valid-message-syntax': 'error',
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
'@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa'] }],
},
},
// Specific rules for XO dev-related files
{
files: [
'@xen-orchestra/lite/src/stories/**/*',
'@xen-orchestra/lite/src/components/component-story/**/*',
'@xen-orchestra/web/src/pages/dev/**/*',
'@xen-orchestra/lite/src/views/story/**/*',
],
rules: {
'@intlify/vue-i18n/no-raw-text': 'off',
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/src/pages/**/*.vue'],
parserOptions: {
sourceType: 'module',
},
rules: {
'vue/multi-word-component-names': 'off',
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
parserOptions: {
sourceType: 'module',
},
rules: {
'eslint-comments/disable-enable-pair': 'off',
'eslint-comments/no-unlimited-disable': 'off',
},
},
],
parserOptions: {
@@ -288,7 +71,7 @@ module.exports = {
// this rule can prevent race condition bugs like parallel `a += await foo()`
//
// as it has a lots of false positive, it is only enabled as a warning for now
'require-atomic-updates': ['warn', { allowProperties: true }],
'require-atomic-updates': 'warn',
strict: 'error',
},

48
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,48 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
assignees: ''
---
1. ⚠️ **If you don't follow this template, the issue will be closed**.
2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
Are you using XOA or XO from the sources?
If XOA:
- which release channel? (`stable` vs `latest`)
- please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
If XO from the sources:
- Provide **your commit number**. If it's older than a week, we won't investigate
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Environment (please provide the following information):**
- Node: [e.g. 16.12.1]
- hypervisor: [e.g. XCP-ng 8.2.0]
**Additional context**
Add any other context about the problem here.

View File

@@ -1,119 +0,0 @@
name: Bug Report
description: Create a report to help us improve
labels: ['type: bug :bug:', 'status: triaging :triangular_flag_on_post:']
body:
- type: markdown
attributes:
value: |
1. ⚠️ **If you don't follow this template, the issue will be closed**.
2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
- type: markdown
attributes:
value: '## Are you using XOA or XO from the sources?'
- type: dropdown
id: xo-origin
attributes:
label: Are you using XOA or XO from the sources?
options:
- XOA
- XO from the sources
- both
validations:
required: false
- type: markdown
attributes:
value: '### If XOA:'
- type: dropdown
id: xoa-channel
attributes:
label: Which release channel?
description: please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
options:
- stable
- latest
- both
validations:
required: false
- type: markdown
attributes:
value: '### If XO from the sources:'
- type: markdown
attributes:
value: |
- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
- type: input
id: xo-sources-commit-number
attributes:
label: Provide your commit number
description: If it's older than a week, we won't investigate
placeholder: e.g. 579f0
validations:
required: false
- type: markdown
attributes:
value: '## Bug description:'
- type: textarea
id: bug-description
attributes:
label: Describe the bug
description: A clear and concise description of what the bug is
validations:
required: true
- type: textarea
id: error-message
attributes:
label: Error message
render: Text
validations:
required: false
- type: textarea
id: steps
attributes:
label: To reproduce
description: 'Steps to reproduce the behavior:'
value: |
1. Go to '...'
2. Click on '...'
3. Scroll down to '...'
4. See error
validations:
required: false
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
description: A clear and concise description of what you expected to happen
validations:
required: false
- type: textarea
id: screenshots
attributes:
label: Screenshots
description: If applicable, add screenshots to help explain your problem
validations:
required: false
- type: markdown
attributes:
value: '## Environment (please provide the following information):'
- type: input
id: node-version
attributes:
label: Node
placeholder: e.g. 16.12.1
validations:
required: true
- type: input
id: hypervisor-version
attributes:
label: Hypervisor
placeholder: e.g. XCP-ng 8.2.0
validations:
required: true
- type: textarea
id: additional-context
attributes:
label: Additional context
description: Add any other context about the problem here
validations:
required: false

View File

@@ -24,14 +24,8 @@ jobs:
cache: 'yarn'
- name: Install project dependencies
run: yarn
- name: Ensure packages are sorted in CHANGELOG.unreleased.md
run: ./scripts/gen-deps-list.js --check-order > /dev/null
- name: Ensure yarn.lock is up-to-date
run: git diff --exit-code yarn.lock
- name: Build the project
run: yarn build
- name: Unit tests
run: yarn test-unit
- name: Lint tests
run: yarn test-lint
- name: Integration tests

4
.gitignore vendored
View File

@@ -30,12 +30,8 @@ pnpm-debug.log.*
yarn-error.log
yarn-error.log.*
.env
*.tsbuildinfo
# code coverage
.nyc_output/
coverage/
.turbo/
# https://node-tap.org/dot-tap-folder/
.tap/

View File

@@ -1,6 +0,0 @@
# Ignore Handlebars files because partials are not currently supported
# https://github.com/prettier/prettier/issues/11834
**/*.hbs
# Generated files
/@xen-orchestra/web/typed-router.d.ts

View File

@@ -33,7 +33,8 @@
"test": "node--test"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^17.0.1",
"tap": "^16.3.0",
"test": "^3.2.1"
}
}

View File

@@ -62,42 +62,6 @@ decorateClass(Foo, {
})
```
### `decorateObject(object, map)`
Decorates an object the same way `decorateClass()` decorates a class:
```js
import { decorateObject } from '@vates/decorate-with'
const object = {
get bar() {
// body
},
set bar(value) {
// body
},
baz() {
// body
},
}
decorateObject(object, {
// getter and/or setter
bar: {
// without arguments
get: lodash.memoize,
// with arguments
set: [lodash.debounce, 150],
},
// method (with or without arguments)
baz: lodash.curry,
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.

View File

@@ -80,42 +80,6 @@ decorateClass(Foo, {
})
```
### `decorateObject(object, map)`
Decorates an object the same way `decorateClass()` decorates a class:
```js
import { decorateObject } from '@vates/decorate-with'
const object = {
get bar() {
// body
},
set bar(value) {
// body
},
baz() {
// body
},
}
decorateObject(object, {
// getter and/or setter
bar: {
// without arguments
get: lodash.memoize,
// with arguments
set: [lodash.debounce, 150],
},
// method (with or without arguments)
baz: lodash.curry,
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.

View File

@@ -14,13 +14,10 @@ function applyDecorator(decorator, value) {
}
exports.decorateClass = exports.decorateMethodsWith = function decorateClass(klass, map) {
return decorateObject(klass.prototype, map)
}
function decorateObject(object, map) {
const { prototype } = klass
for (const name of Object.keys(map)) {
const decorator = map[name]
const descriptor = getOwnPropertyDescriptor(object, name)
const descriptor = getOwnPropertyDescriptor(prototype, name)
if (typeof decorator === 'function' || Array.isArray(decorator)) {
descriptor.value = applyDecorator(decorator, descriptor.value)
} else {
@@ -33,11 +30,10 @@ function decorateObject(object, map) {
}
}
defineProperty(object, name, descriptor)
defineProperty(prototype, name, descriptor)
}
return object
return klass
}
exports.decorateObject = decorateObject
exports.perInstance = function perInstance(fn, decorator, ...args) {
const map = new WeakMap()

View File

@@ -20,7 +20,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "2.1.0",
"version": "2.0.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -29,7 +29,7 @@
"ensure-array": "^1.0.0"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^17.0.1",
"test": "^3.2.1"
}
}

View File

@@ -20,9 +20,6 @@ function assertListeners(t, event, listeners) {
}
t.beforeEach(function (t) {
// work around https://github.com/tapjs/tapjs/issues/998
t.context = {}
t.context.ee = new EventEmitter()
t.context.em = new EventListenersManager(t.context.ee)
})

View File

@@ -38,9 +38,9 @@
"version": "1.0.1",
"scripts": {
"postversion": "npm publish --access public",
"test": "tap --allow-incomplete-coverage"
"test": "tap --branches=72"
},
"devDependencies": {
"tap": "^18.7.0"
"tap": "^16.2.0"
}
}

View File

@@ -1,28 +0,0 @@
Mount a vhd generated by xen-orchestra to filesystem
### Library
```js
import { mount } from 'fuse-vhd'
// return a disposable, see promise-toolbox/Disposable
// unmount automatically when disposable is disposed
// in case of differencing VHD, it mounts the full chain
await mount(handler, diskId, mountPoint)
```
### cli
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
```
After installing the package
```
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`

View File

@@ -1,59 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/fuse-vhd
[![Package Version](https://badgen.net/npm/v/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd) ![License](https://badgen.net/npm/license/@vates/fuse-vhd) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/fuse-vhd)](https://bundlephobia.com/result?p=@vates/fuse-vhd) [![Node compatibility](https://badgen.net/npm/node/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd)
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/fuse-vhd):
```sh
npm install --save @vates/fuse-vhd
```
## Usage
Mount a vhd generated by xen-orchestra to filesystem
### Library
```js
import { mount } from 'fuse-vhd'
// return a disposable, see promise-toolbox/Disposable
// unmount automatically when disposable is disposed
// in case of differencing VHD, it mounts the full chain
await mount(handler, diskId, mountPoint)
```
### cli
From the install folder :
```
cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
```
After installing the package
```
xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
```
remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,26 +0,0 @@
#!/usr/bin/env node
import Disposable from 'promise-toolbox/Disposable'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { mount } from './index.mjs'
async function* main([remoteUrl, vhdPathInRemote, mountPoint]) {
if (mountPoint === undefined) {
throw new TypeError('missing arg: cli <remoteUrl> <vhdPathInRemote> <mountPoint>')
}
const handler = yield getSyncedHandler({ url: remoteUrl })
const mounted = await mount(handler, vhdPathInRemote, mountPoint)
let disposePromise
process.on('SIGINT', async () => {
// ensure single dispose
if (!disposePromise) {
disposePromise = mounted.dispose()
}
await disposePromise
process.exit()
})
}
Disposable.wrap(main)(process.argv.slice(2))

View File

@@ -1,11 +1,8 @@
import LRU from 'lru-cache'
import Fuse from 'fuse-native'
import { createLogger } from '@xen-orchestra/log'
import { VhdSynthetic } from 'vhd-lib'
import { Disposable, fromCallback } from 'promise-toolbox'
const { warn } = createLogger('vates:fuse-vhd')
// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
const stat = st => ({
mtime: st.mtime || new Date(),
@@ -55,16 +52,13 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
},
read(path, fd, buf, len, pos, cb) {
if (path === '/vhd0') {
return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
warn('read error', { path, len, pos, error })
cb(Fuse.EIO)
})
return vhd.readRawData(pos, len, cache, buf).then(cb)
}
throw new Error(`read file ${path} not exists`)
},
})
return new Disposable(
() => fromCallback(cb => fuse.unmount(cb)),
fromCallback(cb => fuse.mount(cb))
() => fromCallback(() => fuse.unmount()),
fromCallback(() => fuse.mount())
)
})

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/fuse-vhd",
"version": "2.1.1",
"version": "2.0.0",
"license": "ISC",
"private": false,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,15 +19,10 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.1.7",
"@xen-orchestra/log": "^0.6.0",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.11.0"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"
"vhd-lib": "^4.8.0"
},
"scripts": {
"postversion": "npm publish --access public"

View File

@@ -1,18 +0,0 @@
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
```js
import { jsonHash } from '@vates/json-hash'
console.log(jsonHash('foo'))
// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
// order of properties is ignored
console.log(jsonHash({ foo: 0, bar: 1 }))
// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
console.log(jsonHash({ bar: 1, foo: 0 }))
// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,51 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/json-hash
[![Package Version](https://badgen.net/npm/v/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash) ![License](https://badgen.net/npm/license/@vates/json-hash) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/json-hash)](https://bundlephobia.com/result?p=@vates/json-hash) [![Node compatibility](https://badgen.net/npm/node/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash)
> Compute a stable hash from a JSON-ifiable value
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/json-hash):
```sh
npm install --save @vates/json-hash
```
## Usage
The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
```js
import { jsonHash } from '@vates/json-hash'
console.log(jsonHash('foo'))
// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
// order of properties is ignored
console.log(jsonHash({ foo: 0, bar: 1 }))
// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
console.log(jsonHash({ bar: 1, foo: 0 }))
// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,38 +0,0 @@
'use strict'
const { createHash } = require('node:crypto')
function updateJsonHash(value, hash) {
if (value !== null && typeof value === 'object') {
if (Array.isArray(value)) {
hash.update('[')
for (const item of value) {
updateJsonHash(item, hash)
// trailing is not a big deal because it does not need to be valid JSON
hash.update(',')
}
hash.update(']')
} else {
hash.update('{')
for (const key of Object.keys(value).sort()) {
updateJsonHash(key, hash)
hash.update(':')
updateJsonHash(value[key], hash)
// trailing is not a big deal because it does not need to be valid JSON
hash.update(',')
}
hash.update('}')
}
} else {
hash.update(JSON.stringify(value))
}
}
exports.jsonHash = function jsonHash(value) {
// this hash does not need to be secure, it just needs to be fast and with low collisions
const hash = createHash('sha256')
updateJsonHash(value, hash)
return hash.digest('base64url')
}

View File

@@ -1,21 +0,0 @@
'use strict'
const { jsonHash } = require('@vates/json-hash')
const assert = require('node:assert/strict')
const test = require('test')
const cases = [
[null, 'dCNOmK_nSY-12vHzasLXiswzlGT5UHA7jAGYkvmCuQs'],
[42, 'c0dctApWjo2ooEXO0RATfhWfiQrE2og7axfcZRs6gEk'],
['foo', 'siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs'],
[[], 'T1PNoYwrqgwDVLtfmj7L5e0Sq02OEbqHPC8RFhICuUU'],
[{}, 'RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o'],
[{ a: 1, b: 2 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
[{ b: 2, a: 1 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
]
for (const [value, hash] of cases) {
test(JSON.stringify(value), function () {
assert.equal(jsonHash(value), hash)
})
}

View File

@@ -1,33 +0,0 @@
{
"private": false,
"name": "@vates/json-hash",
"description": "Compute a stable hash from a JSON-ifiable value",
"keywords": [
"hash",
"json",
"stable"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/json-hash",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/json-hash",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.2.0",
"engines": {
"node": ">=14.18.0"
},
"devDependencies": {
"test": "^3.3.0"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
}
}

View File

@@ -34,6 +34,7 @@ export default class NbdClient {
#exportSize
#waitBeforeReconnect
#readAhead
#readBlockRetries
#reconnectRetry
#connectTimeout
@@ -54,6 +55,7 @@ export default class NbdClient {
connectTimeout = 6e4,
messageTimeout = 6e4,
waitBeforeReconnect = 1e3,
readAhead = 10,
readBlockRetries = 5,
reconnectRetry = 5,
} = {}
@@ -63,6 +65,7 @@ export default class NbdClient {
this.#exportName = exportname
this.#serverCert = cert
this.#waitBeforeReconnect = waitBeforeReconnect
this.#readAhead = readAhead
this.#readBlockRetries = readBlockRetries
this.#reconnectRetry = reconnectRetry
this.#connectTimeout = connectTimeout

View File

@@ -6,10 +6,7 @@ import { createLogger } from '@xen-orchestra/log'
const { warn } = createLogger('vates:nbd-client:multi')
export default class MultiNbdClient {
#clients = []
#nbdConcurrency
#options
#readAhead
#settings
get exportSize() {
return this.#clients[0].exportSize
@@ -17,64 +14,38 @@ export default class MultiNbdClient {
constructor(settings, { nbdConcurrency = 8, readAhead = 16, ...options } = {}) {
this.#readAhead = readAhead
this.#options = options
this.#nbdConcurrency = nbdConcurrency
if (!Array.isArray(settings)) {
settings = [settings]
}
this.#settings = settings
for (let i = 0; i < nbdConcurrency; i++) {
this.#clients.push(
new NbdClient(settings[i % settings.length], { ...options, readAhead: Math.ceil(readAhead / nbdConcurrency) })
)
}
}
/**
*
* open nbdConcurrency connections to NBD servers
* it must obtain at least one connection to succeed
* it tries to spread connections on multiple host
*/
async connect() {
const candidates = [...this.#settings]
const promises = []
const baseOptions = this.#options
const _connect = async () => {
if (candidates.length === 0) {
return
}
// a little bit of randomization to spread the load
const nbdInfo = candidates[Math.floor(Math.random() * candidates.length)]
const client = new NbdClient(nbdInfo, {
...baseOptions,
readAhead: Math.ceil(this.#readAhead / this.#nbdConcurrency),
})
const connectedClients = []
for (const clientId in this.#clients) {
const client = this.#clients[clientId]
try {
await client.connect()
this.#clients.push(client)
connectedClients.push(client)
} catch (err) {
client.disconnect().catch(() => {})
// do not hammer unreachable hosts, once failed, remove from the list
const candidateIndex = candidates.findIndex(({ address }) => address === nbdInfo.address)
if (candidateIndex >= 0) {
// this candidate may have already been deleted by another parallel promise
candidates.splice(candidateIndex, 1)
}
warn(`can't connect to one nbd client`, { err })
// retry with another candidate (if available)
return _connect()
}
}
for (let i = 0; i < this.#nbdConcurrency; i++) {
promises.push(_connect())
}
await Promise.all(promises)
if (this.#clients.length === 0) {
if (connectedClients.length === 0) {
throw new Error(`Fail to connect to any Nbd client`)
}
if (this.#clients.length < this.#nbdConcurrency) {
if (connectedClients.length < this.#clients.length) {
warn(
`incomplete connection by multi Nbd, only ${this.#clients.length} over ${this.#nbdConcurrency} expected clients`
`incomplete connection by multi Nbd, only ${connectedClients.length} over ${
this.#clients.length
} expected clients`
)
this.#clients = connectedClients
}
}

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.1.0",
"version": "3.0.0",
"engines": {
"node": ">=14.0"
},
@@ -24,14 +24,14 @@
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.6.0",
"promise-toolbox": "^0.21.0",
"xen-api": "^4.2.0"
"xen-api": "^2.0.0"
},
"devDependencies": {
"tap": "^18.7.0",
"tap": "^16.3.0",
"tmp": "^0.2.1"
},
"scripts": {
"postversion": "npm publish --access public",
"test-integration": "tap --allow-incomplete-coverage"
"test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
}
}

View File

@@ -78,14 +78,15 @@ async function killNbdKit() {
)
}
test('it works secured network', async tap => {
test('it works with unsecured network', async tap => {
const path = await createTempFile(FILE_SIZE)
let nbdServer = await spawnNbdKit(path)
const connectionSettings = {
address: '127.0.0.1',
exportname: 'MY_SECRET_EXPORT',
cert: `-----BEGIN CERTIFICATE-----
const client = new MultiNbdClient(
{
address: '127.0.0.1',
exportname: 'MY_SECRET_EXPORT',
cert: `-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUeHpQ0IeD6BmP2zgsv3LV3J4BI/EwDQYJKoZIhvcNAQEL
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA1MTcxMzU1MzBaFw0yNDA1
@@ -107,18 +108,12 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
57ABhfYpMlZ4aHjuN1bL
-----END CERTIFICATE-----
`,
}
const invalid = {
address: '500.500.500.500',
port: 0,
exportname: 'nop',
}
// it should work even with some broken servers
const nbdInfos = [connectionSettings, connectionSettings, invalid]
const client = new MultiNbdClient(nbdInfos, {
nbdConcurrency: 4,
readAhead: 2,
})
},
{
nbdConcurrency: 1,
readAhead: 2,
}
)
await client.connect()
tap.equal(client.exportSize, BigInt(FILE_SIZE))
@@ -161,18 +156,3 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
nbdServer.kill()
await fs.unlink(path)
})
test('fails if server does not answer', async () => {
const client = new MultiNbdClient(
{
address: '500.500.500.500',
port: 0,
exportname: 'nop',
},
{
nbdConcurrency: 1,
readAhead: 2,
}
)
await assert.rejects(client.connect())
})

View File

@@ -17,20 +17,7 @@ import https from 'node:https'
import util from 'util'
import soap from 'soap'
import Cookie from 'soap-cookie' // required for session persistence
import { createLogger } from '@xen-orchestra/log'
const { warn } = createLogger('xo:node-vsphere-soap:client')
class VmwareError extends Error {
constructor(rawError) {
super(rawError.body)
const matches = rawError.body?.match(/<faultstring[^>]*>(.*)<\/faultstring>/im)
this.message = matches?.[1] ?? rawError.message
this.name = 'VmwareError'
this.stack = rawError.stack
// not putting the cause since the error failed to be stringified correctly down the road
warn(rawError)
}
}
// Client class
// inherits from EventEmitter
// possible events: connect, error, ready
@@ -44,20 +31,14 @@ export function Client(vCenterHostname, username, password, sslVerify) {
EventEmitter.call(this)
// sslVerify argument handling
// don't use any proxy to connect to the esxi
if (sslVerify) {
this.clientopts = {
request: axios.create({
proxy: false,
}),
}
this.clientopts = {}
} else {
this.clientopts = {
request: axios.create({
httpsAgent: new https.Agent({
rejectUnauthorized: false,
}),
proxy: false,
}),
}
}
@@ -153,9 +134,8 @@ Client.prototype._connect = function () {
self.clientopts,
function (err, client) {
if (err) {
const vErr = new VmwareError(err)
self.emit('error', vErr)
throw vErr
self.emit('error', err)
throw err
}
self.client = client // save client for later use
@@ -164,7 +144,6 @@ Client.prototype._connect = function () {
.runCommand('RetrieveServiceContent', { _this: 'ServiceInstance' })
.once('result', function (result, raw, soapHeader) {
if (!result.returnval) {
// this is a domain/server error return it as is
self.status = 'disconnected'
self.emit('error', raw)
return
@@ -223,7 +202,7 @@ Client.prototype._close = function () {
function _soapErrorHandler(self, emitter, command, args, err) {
err = err || { body: 'general error' }
if (err.body?.match(/session is not authenticated/)) {
if (err.body.match(/session is not authenticated/)) {
self.status = 'disconnected'
process.removeAllListeners('beforeExit')
@@ -235,19 +214,16 @@ function _soapErrorHandler(self, emitter, command, args, err) {
emitter.emit('result', result, raw, soapHeader)
})
.once('error', function (err) {
const vErr = new VmwareError(err)
emitter.emit('error', vErr)
throw vErr
emitter.emit('error', err.body)
throw err
})
} else {
const vErr = new VmwareError(err)
emitter.emit('error', vErr)
throw vErr
emitter.emit('error', err.body)
throw err
}
} else {
const vErr = new VmwareError(err)
emitter.emit('error', vErr)
throw vErr
emitter.emit('error', err.body)
throw err
}
}

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/node-vsphere-soap",
"version": "2.1.0",
"version": "2.0.0",
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
"main": "lib/client.mjs",
"author": "reedog117",
@@ -11,7 +11,6 @@
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/log": "^0.6.0",
"axios": "^1.4.0",
"soap": "^1.0.0",
"soap-cookie": "^0.10.1"

View File

@@ -1,46 +0,0 @@
### `replace(value, replacement)`
> Recursively replace sensitive entries with a replacement value.
```js
import { replace } from '@vates/obfuscate'
replace({ username: 'foo', password: 'bar' }, 'obfuscated')
// → { username: 'foo', password: 'obfuscated'}
```
### `obfuscate(value)`
> Recursively replace sensitive entries with a special tag.
```js
import { obfuscate } from '@vates/obfuscate'
obfuscate({ username: 'foo', password: 'bar' })
// → {
// username: 'foo',
// password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// }
```
### `merge(newValue, oldValue)`
> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
>
> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
```js
import { obfuscate, merge } from '@vates/obfuscate'
const original = { username: 'foo', password: 'bar', token: 'baz' }
// compute an obfuscated value to send to a client
const obfuscated = obfuscate(original)
// let the client update any entries
obfuscated.token = 'qux'
// merge the updated value with the original one
merge(obfuscated, original)
// → { username: 'foo', password: 'bar', token: 'qux' }
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,77 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/obfuscate
[![Package Version](https://badgen.net/npm/v/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate) ![License](https://badgen.net/npm/license/@vates/obfuscate) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/obfuscate)](https://bundlephobia.com/result?p=@vates/obfuscate) [![Node compatibility](https://badgen.net/npm/node/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate)
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/obfuscate):
```sh
npm install --save @vates/obfuscate
```
## Usage
### `replace(value, replacement)`
> Recursively replace sensitive entries with a replacement value.
```js
import { replace } from '@vates/obfuscate'
replace({ username: 'foo', password: 'bar' }, 'obfuscated')
// → { username: 'foo', password: 'obfuscated'}
```
### `obfuscate(value)`
> Recursively replace sensitive entries with a special tag.
```js
import { obfuscate } from '@vates/obfuscate'
obfuscate({ username: 'foo', password: 'bar' })
// → {
// username: 'foo',
// password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
// }
```
### `merge(newValue, oldValue)`
> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
>
> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
```js
import { obfuscate, merge } from '@vates/obfuscate'
const original = { username: 'foo', password: 'bar', token: 'baz' }
// compute an obfuscated value to send to a client
const obfuscated = obfuscate(original)
// let the client update any entries
obfuscated.token = 'qux'
// merge the updated value with the original one
merge(obfuscated, original)
// → { username: 'foo', password: 'bar', token: 'qux' }
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,23 +0,0 @@
{
"private": false,
"name": "@vates/obfuscate",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/obfuscate",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/obfuscate",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.1.0",
"engines": {
"node": ">=8.10"
},
"scripts": {
"postversion": "npm publish --access public"
}
}

View File

@@ -9,15 +9,6 @@ function assert(name, value) {
}
}
const WHITESPACES = /\s+/g
function cleanToken(otp) {
// remove whitespaces
//
// some OTP clients add whitespaces for better readability (e.g. `XXX XXX`)
// and users might be tempted to write them
return otp && otp.replace(WHITESPACES, '')
}
// https://github.com/google/google-authenticator/wiki/Key-Uri-Format
function generateUri(protocol, label, params) {
assert('label', typeof label === 'string')
@@ -73,7 +64,7 @@ export function generateHotpUri({ counter, digits, issuer, label, secret }) {
}
export async function verifyHotp(token, opts) {
return cleanToken(token) === (await generateHotp(opts))
return token === (await generateHotp(opts))
}
function totpCounter(period = 30, timestamp = Math.floor(Date.now() / 1e3)) {
@@ -94,7 +85,6 @@ export async function verifyTotp(token, { period, timestamp, window = 1, ...opts
const counter = totpCounter(period, timestamp)
const end = counter + window
opts.counter = counter - window
token = cleanToken(token)
while (opts.counter <= end) {
if (token === (await generateHotp(opts))) {
return true

View File

@@ -1,5 +1,5 @@
import { strict as assert } from 'node:assert'
import test from 'test'
import { describe, it } from 'tap/mocha'
import {
generateHotp,
@@ -11,8 +11,6 @@ import {
verifyTotp,
} from './index.mjs'
const { describe, it } = test
describe('generateSecret', function () {
it('generates a string of 32 chars', async function () {
const secret = generateSecret()

View File

@@ -22,7 +22,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "1.1.0",
"version": "1.0.0",
"engines": {
"node": ">=15"
},
@@ -31,9 +31,9 @@
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.3.0"
}
}

View File

@@ -0,0 +1,14 @@
'use strict'
const ms = require('ms')
exports.parseDuration = value => {
if (typeof value === 'number') {
return value
}
const duration = ms(value)
if (duration === undefined) {
throw new TypeError(`not a valid duration: ${value}`)
}
return duration
}

View File

@@ -1,22 +0,0 @@
import { describe, it } from 'node:test'
import { parseDuration } from '@vates/parse-duration'
import assert from 'node:assert/strict'
describe('parseDuration()', () => {
it('should parse string', () => {
const input = '2 days'
const expected = 172800000
assert.strictEqual(parseDuration(input), expected)
})
it('should return its input if already a number', () => {
const input = 172800000
assert.strictEqual(parseDuration(input), input)
})
for (const input of [undefined, '', 'invalid duration']) {
it('should throw an error for ' + input, () => {
assert.throws(() => parseDuration(input), { message: `not a valid duration: ${input}` })
})
}
})

View File

@@ -27,12 +27,6 @@
"ms": "^2.1.2"
},
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"prepublishOnly": "yarn run build",
"pretest": "yarn run build",
"test": "node --test"
},
"main": "dist/index.js"
"postversion": "npm publish --access public"
}
}

View File

@@ -1,22 +0,0 @@
'use strict'
const ms = require('ms')
exports.parseDuration = (value: number | string): number => {
const throwError = () => {
throw new TypeError(`not a valid duration: ${value}`)
}
if (typeof value === 'number') {
return value
}
try {
const duration: number | undefined = ms(value)
if (typeof duration !== 'number') {
return throwError()
}
return duration
} catch (err) {
return throwError()
}
}

View File

@@ -1,18 +0,0 @@
{
"compilerOptions": {
"esModuleInterop": true,
"module": "commonjs",
"moduleResolution": "node",
"newLine": "lf",
"noImplicitAny": true,
"strict": false,
"skipLibCheck": true,
"baseUrl": ".",
"sourceMap": false,
"strictNullChecks": true,
"target": "ES2015",
"outDir": "./dist",
"rootDir": "./src"
},
"exclude": ["node_modules", "dist"]
}

View File

@@ -1,7 +1,7 @@
'use strict'
const assert = require('assert/strict')
const { describe, it } = require('test')
const { describe, it } = require('tap').mocha
const { every, not, some } = require('./')

View File

@@ -32,9 +32,9 @@
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.0.1"
}
}

View File

@@ -72,27 +72,6 @@ Task.warning(message, data)
Task.set(property, value)
```
### `Task.run([opts], fn, ...args)`
This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
```js
// options can be passed
await Task.run({ properties: { name: 'foo' } }, fn)
// arguments can be passed
await Task.run(fn, 'foo', 'bar')
// context (this) is forwarded
await Task.run.call(object, object.method)
// as a convenience, you can pass a method name directly
await Task.run.call(object, 'method')
// everything together
await Task.run.call(object, 'method', 'foo', 'bar')
```
### `combineEvents`
Create a consolidated log from individual events.

View File

@@ -88,27 +88,6 @@ Task.warning(message, data)
Task.set(property, value)
```
### `Task.run([opts], fn, ...args)`
This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
```js
// options can be passed
await Task.run({ properties: { name: 'foo' } }, fn)
// arguments can be passed
await Task.run(fn, 'foo', 'bar')
// context (this) is forwarded
await Task.run.call(object, object.method)
// as a convenience, you can pass a method name directly
await Task.run.call(object, 'method')
// everything together
await Task.run.call(object, 'method', 'foo', 'bar')
```
### `combineEvents`
Create a consolidated log from individual events.

View File

@@ -3,10 +3,6 @@
const assert = require('node:assert').strict
const { AsyncLocalStorage } = require('node:async_hooks')
function alreadyEnded() {
throw new Error('task has already ended')
}
// define a read-only, non-enumerable, non-configurable property
function define(object, property, value) {
Object.defineProperty(object, property, { value })
@@ -25,7 +21,7 @@ const asyncStorage = global[asyncStorageKey] ?? (global[asyncStorageKey] = new A
const getTask = () => asyncStorage.getStore()
class Task {
exports.Task = class Task {
static get abortSignal() {
const task = getTask()
if (task !== undefined) {
@@ -33,25 +29,29 @@ class Task {
}
}
static run(...args) {
let opts = args[0]
let fn
if (typeof opts === 'object') {
args.shift()
fn = args[0]
} else {
fn = opts
opts = undefined
static info(message, data) {
const task = getTask()
if (task !== undefined) {
task.#emit('info', { data, message })
}
args.shift()
}
const thisArg = this !== Task ? this : undefined
static run(opts, fn) {
return new this(opts).run(fn)
}
if (typeof fn !== 'function') {
fn = this[fn]
static set(name, value) {
const task = getTask()
if (task !== undefined) {
task.#emit('property', { name, value })
}
}
return new Task(opts).run(() => fn.apply(thisArg, args))
static warning(message, data) {
const task = getTask()
if (task !== undefined) {
task.#emit('warning', { data, message })
}
}
static wrap(opts, fn) {
@@ -110,7 +110,9 @@ class Task {
this.#emit('abortionRequested', { reason: signal.reason })
if (!this.#running) {
this.#end(FAILURE, signal.reason)
const status = FAILURE
this.#status = status
this.#emit('end', { result: signal.reason, status })
}
}
})
@@ -121,26 +123,12 @@ class Task {
}
#emit(type, data) {
assert.equal(this.#startData, undefined, 'task has not started yet')
data.id = this.id
data.timestamp = Date.now()
data.type = type
this.#onProgress(data)
}
#end(status, result) {
assert.equal(this.#status, PENDING)
this.#status = status
this.#emit('end', { status, result })
this.#onProgress = alreadyEnded
}
info(message, data) {
this.#emit('info', { data, message })
}
#maybeStart() {
const startData = this.#startData
if (startData !== undefined) {
@@ -152,7 +140,8 @@ class Task {
async run(fn) {
const result = await this.runInside(fn)
if (this.status === PENDING) {
this.#end(SUCCESS, result)
this.#status = SUCCESS
this.#emit('end', { status: SUCCESS, result })
}
return result
}
@@ -166,27 +155,18 @@ class Task {
this.#maybeStart()
try {
const result = await asyncStorage.run(this, fn, this)
const result = await asyncStorage.run(this, fn)
this.#running = false
return result
} catch (result) {
this.#end(FAILURE, result)
const status = FAILURE
this.#status = status
this.#emit('end', { status, result })
throw result
}
}
set(name, value) {
assert.equal(this.status, PENDING)
this.#emit('property', { name, value })
}
warning(message, data) {
assert.equal(this.status, PENDING)
this.#emit('warning', { data, message })
}
wrap(fn) {
const task = this
return function taskRun() {
@@ -201,15 +181,3 @@ class Task {
}
}
}
exports.Task = Task
// setup static aliases
for (const name of ['info', 'set', 'warning']) {
const method = Task.prototype[name]
Task[name] = function () {
const task = getTask()
if (task !== undefined) {
return method.apply(task, arguments)
}
}
}

View File

@@ -3,7 +3,6 @@
const assert = require('node:assert').strict
const { describe, it } = require('test')
const { makeOnProgress } = require('./combineEvents.js')
const { Task } = require('./index.js')
const noop = Function.prototype
@@ -163,44 +162,6 @@ describe('Task', function () {
})
})
describe('.run()', function () {
function fn(...args) {
Task.set('baz', 'qux')
return [this, ...args]
}
const obj = { fn }
it('accepts optional options', async function () {
assert.deepEqual(
await Task.run(
{
properties: { foo: 'bar' },
onProgress: makeOnProgress({
onRootTaskEnd(log) {
assert.deepEqual(log.properties, { __proto__: null, foo: 'bar', baz: 'qux' })
},
}),
},
fn
),
[undefined]
)
})
it('accepts optional arguments', async function () {
assert.deepEqual(await Task.run(fn, 'foo', 'bar'), [undefined, 'foo', 'bar'])
})
it('accepts optional context', async function () {
assert.deepEqual(await Task.run.call('foo', fn), ['foo'])
})
it('accepts a method name instead of a function', async function () {
assert.deepEqual(await Task.run.call(obj, 'fn'), [obj])
})
})
describe('.set()', function () {
it('does nothing when run outside a task', function () {
Task.set('progress', 10)
@@ -219,31 +180,6 @@ describe('Task', function () {
})
})
describe('#set()', function () {
const name = 'progress'
const value = 10
it('throws when the task is not started', function () {
const task = createTask()
assert.throws(() => task.set(name, value), { message: 'task has not started yet' })
})
it(`emits an property message`, async function () {
const task = createTask()
await task.run(async () => {
await Task.run(() => {
task.set(name, value)
assertEvent(task, {
name,
type: 'property',
value,
})
})
})
})
})
describe('.warning()', function () {
it('does nothing when run outside a task', function () {
Task.warning('foo')
@@ -262,30 +198,6 @@ describe('Task', function () {
})
})
for (const type of ['info', 'warning']) {
describe(`#${type}()`, function () {
it('throws when the task is not started', function () {
const task = createTask()
assert.throws(() => task[type]('foo'), { message: 'task has not started yet' })
})
it(`emits an ${type} message`, async function () {
const task = createTask()
await task.run(async () => {
await Task.run(() => {
task[type]('foo')
assertEvent(task, {
data: undefined,
message: 'foo',
type,
})
})
})
})
})
}
describe('#id', function () {
it('can be set', function () {
const task = createTask()

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.4.0",
"version": "0.2.0",
"engines": {
"node": ">=14"
},

View File

@@ -1,128 +0,0 @@
### Formatting
```js
import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// → {
// name: 'methodCall',
// children: [
// {
// name: 'methodName',
// children: [ 'examples.getStateName' ]
// },
// {
// name: 'params',
// children: [
// {
// name: 'param',
// children: [ { name: 'value', children: [ '40' ] } ]
// }
// ]
// }
// ]
// }
// There is a dedicated method to format a JS value
xmlRpcFormatter.format_js(new Date(900684535000))
// → {
// name: 'dateTime.iso8601',
// children: [ '1998-07-17T14:08:55.000Z' ]
// }
```
Formatting XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { formatXml } from '@vates/xml/format'
// it can now be passed to the XML-RPC parser
const tree = xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
const xml = formatXml(tree, { indent: 0 })
```
### Parsing
```js
import { xmlRpcParser } from '@vates/xml-rpc/parser'
xmlRpcParser.parse({
name: 'methodResponse',
children: [
{
name: 'params',
children: [
{
name: 'param',
children: [
{
name: 'value',
children: [
{
name: 'string',
children: ['South Dakota'],
},
],
},
],
},
],
},
],
})
// { params: [ 'South Dakota' ] }
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcParser.parse_array({
name: 'array',
children: [
{
name: 'data',
children: [
{
name: 'value',
children: [{ name: 'boolean', children: ['1'] }],
},
{
name: 'value',
children: [{ name: 'int', children: ['42'] }],
},
],
},
],
})
// [ true, 42 ]
```
Parsing XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { parseXml } from '@vates/xml/parse'
const xml = `
<?xml version="1.0"?>
<methodResponse>
<params>
<param>
<value><string>South Dakota</string></value>
</param>
</params>
</methodResponse>
`
const tree = parseXml(xml)
// it can now be passed to the XML-RPC parser
xmlRpcParser.parse(tree)
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,161 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/xml-rpc
[![Package Version](https://badgen.net/npm/v/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc) ![License](https://badgen.net/npm/license/@vates/xml-rpc) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml-rpc)](https://bundlephobia.com/result?p=@vates/xml-rpc) [![Node compatibility](https://badgen.net/npm/node/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc)
> Parse and format XML-RPC messages, do not handle transport or XML
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/xml-rpc):
```sh
npm install --save @vates/xml-rpc
```
## Usage
### Formatting
```js
import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// → {
// name: 'methodCall',
// children: [
// {
// name: 'methodName',
// children: [ 'examples.getStateName' ]
// },
// {
// name: 'params',
// children: [
// {
// name: 'param',
// children: [ { name: 'value', children: [ '40' ] } ]
// }
// ]
// }
// ]
// }
// There is a dedicated method to format a JS value
xmlRpcFormatter.format_js(new Date(900684535000))
// → {
// name: 'dateTime.iso8601',
// children: [ '1998-07-17T14:08:55.000Z' ]
// }
```
Formatting XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { formatXml } from '@vates/xml/format'
// it can now be passed to the XML-RPC parser
const tree = xmlRpcFormatter.format_methodCall({
methodName: 'examples.getStateName',
params: [40],
})
// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
const xml = formatXml(tree, { indent: 0 })
```
### Parsing
```js
import { xmlRpcParser } from '@vates/xml-rpc/parser'
xmlRpcParser.parse({
name: 'methodResponse',
children: [
{
name: 'params',
children: [
{
name: 'param',
children: [
{
name: 'value',
children: [
{
name: 'string',
children: ['South Dakota'],
},
],
},
],
},
],
},
],
})
// { params: [ 'South Dakota' ] }
// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
xmlRpcParser.parse_array({
name: 'array',
children: [
{
name: 'data',
children: [
{
name: 'value',
children: [{ name: 'boolean', children: ['1'] }],
},
{
name: 'value',
children: [{ name: 'int', children: ['42'] }],
},
],
},
],
})
// [ true, 42 ]
```
Parsing XML is out of this library's scope, but you can achieve it easily with
`@vates/xml`:
```js
import { parseXml } from '@vates/xml/parse'
const xml = `
<?xml version="1.0"?>
<methodResponse>
<params>
<param>
<value><string>South Dakota</string></value>
</param>
</params>
</methodResponse>
`
const tree = parseXml(xml)
// it can now be passed to the XML-RPC parser
xmlRpcParser.parse(tree)
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,104 +0,0 @@
'use strict'
function node(name, attributes, children) {
if (Array.isArray(attributes)) {
children = attributes
attributes = undefined
}
return { name, attributes, children }
}
class XmlRpcFormatter {
format_js(value) {
const type = typeof value
if (type === 'object') {
return value === null
? this.format_nil()
: value instanceof Date
? this['format_dateTime.iso8601'](value)
: Array.isArray(value)
? this.format_array(value)
: Buffer.isBuffer(value)
? this.format_base64(value)
: this.format_struct(value)
}
return this['format_' + type](value)
}
format_array(values) {
return node('array', [node('data', values.map(this.format_value, this))])
}
format_base64(buf) {
return node('base64', [buf.toString('base64')])
}
format_boolean(bool) {
return node('boolean', [bool ? '1' : '0'])
}
'format_dateTime.iso8601'(date) {
return node('dateTime.iso8601', [date.toISOString()])
}
format_double(str) {
return node('double', [String(str)])
}
format_fault(val) {
return node('fault', [this.format_value(val)])
}
format_i4(str) {
return node('i4', [String(str)])
}
format_int(str) {
return node('int', [String(str)])
}
format_member([name, value]) {
return node('member', [node('name', [name]), this.format_value(value)])
}
format_methodCall({ methodName, params }) {
return node('methodCall', [node('methodName', [methodName]), this.format_params(params)])
}
format_methodResponse({ fault, params }) {
return node('methodResponse', [fault !== undefined ? this.format_fault(fault) : this.format_params(params)])
}
format_nil() {
return node('nil')
}
format_number(num) {
return Number.isInteger(num) ? this.format_int(num) : this.format_double(num)
}
format_param(value) {
return node('param', [this.format_value(value)])
}
format_params(params) {
return node('params', params.map(this.format_param, this))
}
format_string(str) {
return str
}
format_struct(struct) {
return node('struct', Object.entries(struct).map(this.format_member, this))
}
format_value(value) {
return node('value', [this.format_js(value)])
}
}
exports.Formatter = XmlRpcFormatter
exports.xmlRpcFormatter = new XmlRpcFormatter()

View File

@@ -1,33 +0,0 @@
{
"private": false,
"name": "@vates/xml-rpc",
"description": "Parse and format XML-RPC messages, do not handle transport or XML",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml-rpc",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/xml-rpc",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "1.0.0",
"engines": {
"node": ">=12"
},
"devDependencies": {
"@vates/xml": "^2.0.0",
"test": "^3.3.0"
},
"exports": {
"./formatter": "./formatter.js",
"./parser": "./parser.js"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
}
}

View File

@@ -1,133 +0,0 @@
'use strict'
function toValue(node) {
const obj = { __proto__: null }
for (const child of node.children) {
if (typeof child === 'string') {
return child
}
const { name } = child
if (Object.hasOwn(obj, name)) {
const prev = obj[name]
if (Array.isArray(obj, name)) {
prev.push(child)
} else {
obj[name] = [prev, child]
}
} else {
obj[name] = child
}
}
return obj
}
// https://en.wikipedia.org/wiki/XML-RPC#Data_types
class XmlRpcParser {
parse(node) {
return this['parse_' + node.name](node)
}
parse_array({ children: [data] }) {
return data.children.map(this.parse_value, this)
}
parse_base64({ children: [str] }) {
return Buffer.from(str, 'base64')
}
parse_boolean({ children: [str] }) {
return Boolean(Number(str))
}
'parse_dateTime.iso8601'({ children: [str] }) {
const date = new Date(str)
// date successfully parsed
if (!Number.isNaN(+date)) {
return date
}
// XML-RPC dates may use YYYYMMDD instead of YYYY-MM-DD which ECMAScript
// does not parse
return new Date(str.slice(0, 4) + '-' + str.slice(4, 6) + '-' + str.slice(6))
}
parse_double({ children: [str] }) {
return Number(str)
}
parse_fault({ children: [value] }) {
return this.parse_value(value)
}
parse_i4({ children: [str] }) {
return Number(str)
}
parse_int({ children: [str] }) {
return Number(str)
}
parse_member(member) {
member = toValue(member)
return [member.name.children[0], this.parse_value(member.value)]
}
parse_methodCall(methodCall) {
methodCall = toValue(methodCall)
return {
methodName: methodCall.methodName.children[0],
params: this.parse_params(methodCall.params),
}
}
parse_methodResponse(methodResponse) {
methodResponse = toValue(methodResponse)
const { fault } = methodResponse
if (fault !== undefined) {
return { fault: this.parse_fault(fault) }
}
return { params: this.parse_params(methodResponse.params) }
}
parse_nil() {
return null
}
parse_param(param) {
return this.parse_value(param.children[0])
}
parse_params(params) {
return params.children.map(this.parse_param, this)
}
parse_string({ children: [str] }) {
return str
}
parse_struct(struct) {
return Object.fromEntries(struct.children.map(this.parse_member, this))
}
parse_value({ children }) {
if (children.length === 0) {
return ''
}
const child = children[0]
// can be a plain string
if (typeof child === 'string') {
return child
}
return this.parse(child)
}
}
exports.XmlRpcParser = XmlRpcParser
exports.xmlRpcParser = new XmlRpcParser()

View File

@@ -1,19 +0,0 @@
'use strict'
const { formatXml } = require('@vates/xml/format')
const { join } = require('node:path')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { xmlRpcFormatter } = require('@vates/xml-rpc/formatter')
const assert = require('node:assert/strict')
const test = require('test')
const methodCall = require('./methodCall.js')
test(function () {
const xmlTree = parseXml(readFileSync(join(__dirname, 'methodCall.xml')))
const xmlRpc = xmlRpcFormatter.format_methodCall(methodCall)
assert.deepEqual(parseXml(formatXml(xmlRpc)), xmlTree)
})

View File

@@ -1,16 +0,0 @@
'use strict'
module.exports = {
methodName: 'examples.getStateName',
params: [
[1404, 'Something here'],
Buffer.from([121, 111, 117, 32, 99, 97, 110, 39, 116, 32, 114, 101, 97, 100, 32, 116, 104, 105, 115, 33]),
true,
new Date('1998-07-17T14:08:55.000Z'),
-12.53,
42,
'Hello world!',
{ foo: 1, bar: 2 },
null,
],
}

View File

@@ -1,73 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<methodCall>
<methodName>examples.getStateName</methodName>
<params>
<param>
<value>
<array>
<data>
<value>
<int>1404</int>
</value>
<value>
Something here
</value>
</data>
</array>
</value>
</param>
<param>
<value>
<base64>eW91IGNhbid0IHJlYWQgdGhpcyE=</base64>
</value>
</param>
<param>
<value>
<boolean>1</boolean>
</value>
</param>
<param>
<value>
<dateTime.iso8601>1998-07-17T14:08:55.000Z</dateTime.iso8601>
</value>
</param>
<param>
<value>
<double>-12.53</double>
</value>
</param>
<param>
<value>
<int>42</int>
</value>
</param>
<param>
<value>
Hello world!
</value>
</param>
<param>
<value>
<struct>
<member>
<name>foo</name>
<value>
<int>1</int>
</value>
</member>
<member>
<name>bar</name>
<value>
<int>2</int>
</value>
</member>
</struct>
</value>
</param>
<param>
<value>
<nil />
</value>
</param>
</params>
</methodCall>

View File

@@ -1,32 +0,0 @@
'use strict'
const { join } = require('node:path')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { xmlRpcParser } = require('@vates/xml-rpc/parser')
const assert = require('node:assert/strict')
const test = require('test')
const methodCall = require('./methodCall.js')
test('parser.parse()', function () {
const xml = String(readFileSync(join(__dirname, 'methodCall.xml')))
const xmlRpc = parseXml(xml)
assert.deepEqual(xmlRpcParser.parse(xmlRpc), methodCall)
})
test('datetime without dashes nor milliseconds', function () {
assert.deepEqual(
xmlRpcParser.parse({ name: 'dateTime.iso8601', children: ['19980717T14:08:55Z'] }),
new Date('1998-07-17T14:08:55Z')
)
})
test('i4 support', function () {
assert.equal(xmlRpcParser.parse({ name: 'i4', children: ['42'] }), 42)
})
test('string support', function () {
assert.equal(xmlRpcParser.parse({ name: 'string', children: ['foo'] }), 'foo')
})

View File

@@ -1,56 +0,0 @@
### `parseXml(xml, [opts])`
> Based on [`sax`](https://www.npmjs.com/package/sax)
`opts` is an optional object which can contain the following options:
- `normalize = true`: if true, then turn any whitespace into a single space
- `strict = true`: whether or not to be a jerk
- `trim = true`: whether or not to trim text nodes
```js
import { parseXml } from '@xen-orchestra/xml/parse'
const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>
`)
// → {
// name: 'tag1',
// attributes: { attr1: 'value1', attr2: 'value2' },
// children: [
// 'Text & entities',
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'ns:tag3', attributes: {}, children: [] }
// ]
// }
```
### `formatXml(tree, [opts])`
`opts` is an optional object which can contain the following options:
- `includeDeclaration = true`: whether to include an XML declaration
- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
```js
import { formatXml } from '@xen-orchestra/xml/format'
formatXml({
name: 'tag1',
attributes: { attr1: 'value1', attr2: 'value2' },
children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
})
// → <?xml version="1.0" encoding="UTF-8"?>
// <tag1 attr1="value1" attr2="value2">
// Text &amp; entities
// <tag2 />
// <tag2 />
// <ns:tag3 />
// </tag1>
```

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,89 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/xml
[![Package Version](https://badgen.net/npm/v/@vates/xml)](https://npmjs.org/package/@vates/xml) ![License](https://badgen.net/npm/license/@vates/xml) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml)](https://bundlephobia.com/result?p=@vates/xml) [![Node compatibility](https://badgen.net/npm/node/@vates/xml)](https://npmjs.org/package/@vates/xml)
> Simple XML formatting/parsing
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/xml):
```sh
npm install --save @vates/xml
```
## Usage
### `parseXml(xml, [opts])`
> Based on [`sax`](https://www.npmjs.com/package/sax)
`opts` is an optional object which can contain the following options:
- `normalize = true`: if true, then turn any whitespace into a single space
- `strict = true`: whether or not to be a jerk
- `trim = true`: whether or not to trim text nodes
```js
import { parseXml } from '@xen-orchestra/xml/parse'
const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>
`)
// → {
// name: 'tag1',
// attributes: { attr1: 'value1', attr2: 'value2' },
// children: [
// 'Text & entities',
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'tag2', attributes: {}, children: [] },
// { name: 'ns:tag3', attributes: {}, children: [] }
// ]
// }
```
### `formatXml(tree, [opts])`
`opts` is an optional object which can contain the following options:
- `includeDeclaration = true`: whether to include an XML declaration
- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
```js
import { formatXml } from '@xen-orchestra/xml/format'
formatXml({
name: 'tag1',
attributes: { attr1: 'value1', attr2: 'value2' },
children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
})
// → <?xml version="1.0" encoding="UTF-8"?>
// <tag1 attr1="value1" attr2="value2">
// Text &amp; entities
// <tag2 />
// <tag2 />
// <ns:tag3 />
// </tag1>
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,30 +0,0 @@
#!/usr/bin/env node
'use strict'
const { inspect } = require('node:util')
const { formatXml } = require('@vates/xml/format')
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
function log(val) {
process.stdout.write(inspect(val, false, null, true))
process.stdout.write('\n')
}
function main([inputPath = 0]) {
const input = readFileSync(inputPath)
// attempt to parse from JSON
let data
try {
data = JSON.parse(input)
} catch (error) {
// fallback to XML
log(parseXml(input))
return
}
process.stdout.write(formatXml(data))
}
main(process.argv.slice(2))

View File

@@ -1,64 +0,0 @@
'use strict'
const ENTITIES = {
'"': '&quot;',
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
"'": '&apos;',
}
const RE = new RegExp(Object.keys(ENTITIES).join('|'), 'g')
function replace(match) {
return ENTITIES[match]
}
function escape(str) {
return str.replace(RE, replace)
}
function formatNode(node, depth) {
const indent = this.indent.repeat(depth)
if (typeof node === 'object') {
const line = [indent, '<', node.name]
const { attributes } = node
if (attributes !== undefined) {
for (const name of Object.keys(attributes)) {
line.push(' ', name, '="', escape(attributes[name]), '"')
}
}
const { children } = node
if (children === undefined || children.length === 0) {
line.push(' />')
this.lines.push(line.join(''))
} else {
line.push('>')
this.lines.push(line.join(''))
for (const child of children) {
formatNode.call(this, child, depth + 1)
}
this.lines.push(indent + '</' + node.name + '>')
}
} else {
// string (or scalar)
this.lines.push(indent + escape(String(node)))
}
}
exports.formatXml = function formatXml(tree, { includeDeclaration = true, indent = 2 } = {}) {
const lines = []
if (includeDeclaration) {
lines.push('<?xml version="1.0" encoding="UTF-8"?>')
}
const ctx = {
lines,
indent: typeof indent === 'number' ? ' '.repeat(indent) : indent,
}
formatNode.call(ctx, tree, 0)
return lines.join(indent !== 0 && indent !== '' ? '\n' : '')
}

View File

@@ -1,35 +0,0 @@
{
"private": false,
"name": "@vates/xml",
"description": "Simple XML formatting/parsing",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/xml",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "2.0.0",
"engines": {
"node": ">=14"
},
"dependencies": {
"sax": "^1.3.0"
},
"devDependencies": {
"test": "^3.3.0"
},
"exports": {
"./format": "./format.js",
"./parse": "./parse.js"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
}
}

View File

@@ -1,23 +0,0 @@
'use strict'
const sax = require('sax')
exports.parseXml = function parseXml(xml, { normalize = true, strict = true, trim = true } = {}) {
const stack = [{ children: [] }]
const parser = sax.parser(strict, { normalize, trim })
parser.ontext = text => {
stack[stack.length - 1].children.push(text)
}
parser.onopentag = ({ name, attributes }) => {
stack.push({ name, attributes, children: [] })
}
parser.onclosetag = () => {
const node = stack.pop()
stack[stack.length - 1].children.push(node)
}
parser.write(xml).close()
return stack[0].children[0]
}

View File

@@ -1,25 +0,0 @@
{
"name": "tag1",
"attributes": {
"attr1": "value1",
"attr2": "value2"
},
"children": [
"Text & entities",
{
"name": "tag2",
"attributes": {},
"children": []
},
{
"name": "tag2",
"attributes": {},
"children": []
},
{
"name": "ns:tag3",
"attributes": {},
"children": []
}
]
}

View File

@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<tag1 attr1="value1" attr2="value2">
Text &amp; entities
<tag2 />
<tag2 />
<ns:tag3 />
</tag1>

View File

@@ -1,18 +0,0 @@
'use strict'
const { formatXml } = require('@vates/xml/format')
const { join } = require('node:path')
const { readFileSync } = require('node:fs')
const assert = require('node:assert/strict')
const test = require('test')
test('format()', function () {
const xml = String(readFileSync(join(__dirname, 'data.xml')))
const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
assert.equal(formatXml(tree) + '\n', xml)
})
test('supports missing attributes and children', function () {
assert.equal(formatXml({ name: 'foo' }, { includeDeclaration: false }), '<foo />')
})

View File

@@ -1,14 +0,0 @@
'use strict'
const { parseXml } = require('@vates/xml/parse')
const { readFileSync } = require('node:fs')
const { join } = require('node:path')
const assert = require('node:assert/strict')
const test = require('test')
test('parse()', function () {
const xml = String(readFileSync(join(__dirname, 'data.xml')))
const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
assert.deepEqual(parseXml(xml), tree)
})

View File

@@ -35,7 +35,7 @@
"test": "node--test"
},
"devDependencies": {
"sinon": "^18.0.0",
"sinon": "^17.0.1",
"test": "^3.2.1"
}
}

View File

@@ -130,10 +130,6 @@ class AuditCore {
return nValid
}
get(id) {
return this._storage.get(id)
}
async *getFrom(newest) {
const storage = this._storage

View File

@@ -1,7 +1,7 @@
'use strict'
const assert = require('assert/strict')
const { afterEach, describe, it } = require('test')
const { afterEach, describe, it } = require('tap').mocha
const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')

View File

@@ -7,16 +7,16 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.3.0",
"version": "0.2.3",
"engines": {
"node": ">=14"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
"test": "tap --lines 67 --functions 92 --branches 52 --statements 67"
},
"dependencies": {
"@vates/decorate-with": "^2.1.0",
"@vates/decorate-with": "^2.0.0",
"@xen-orchestra/log": "^0.6.0",
"golike-defer": "^0.5.1",
"object-hash": "^2.0.1"
@@ -28,6 +28,6 @@
"url": "https://vates.fr"
},
"devDependencies": {
"test": "^3.3.0"
"tap": "^16.0.1"
}
}

View File

@@ -7,8 +7,8 @@
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.53.1",
"@xen-orchestra/fs": "^4.1.7",
"@xen-orchestra/backups": "^0.44.3",
"@xen-orchestra/fs": "^4.1.3",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",
"lodash": "^4.17.15",
@@ -27,7 +27,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "1.0.23",
"version": "1.0.14",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -160,10 +160,10 @@ export class ImportVmBackup {
// update the stream with the negative vhd stream
stream = await negativeVhd.stream()
vdis[vdiRef].baseVdi = snapshotCandidate
} catch (error) {
} catch (err) {
// can be a broken VHD chain, a vhd chain with a key backup, ....
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
warn(`can't use differential restore`, { error })
warn(`can't use differential restore`, err)
disposableDescendants?.dispose()
}
}
@@ -191,7 +191,7 @@ export class ImportVmBackup {
async #decorateIncrementalVmMetadata() {
const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
const ignoredVdis = new Set(
Object.entries(mapVdisSrs)
.filter(([_, srUuid]) => srUuid === null)

View File

@@ -18,7 +18,6 @@ import fromEvent from 'promise-toolbox/fromEvent'
import groupBy from 'lodash/groupBy.js'
import pDefer from 'promise-toolbox/defer'
import pickBy from 'lodash/pickBy.js'
import reduce from 'lodash/reduce.js'
import tar from 'tar'
import zlib from 'zlib'
@@ -36,8 +35,6 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -192,14 +189,13 @@ export class RemoteAdapter {
// check if we will be allowed to merge a a vhd created in this adapter
// with the vhd at path `path`
async isMergeableParent(packedParentUid, path) {
return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
return await Disposable.use(openVhd(this.handler, path), vhd => {
// this baseUuid is not linked with this vhd
if (!vhd.footer.uuid.equals(packedParentUid)) {
return false
}
// check if all the chain is composed of vhd directory
const isVhdDirectory = vhd.checkVhdsClass(VhdDirectory)
const isVhdDirectory = vhd instanceof VhdDirectory
return isVhdDirectory
? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
: !this.useVhdDirectory()
@@ -753,37 +749,10 @@ export class RemoteAdapter {
}
async readVmBackupMetadata(path) {
let json
let isImmutable = false
let remoteIsImmutable = false
// if the remote is immutable, check if this metadatas are also immutables
try {
// this file is not encrypted
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
remoteIsImmutable = true
} catch (error) {
if (error.code !== 'ENOENT') {
throw error
}
}
try {
// this will trigger an EPERM error if the file is immutable
json = await this.handler.readFile(path, { flag: 'r+' })
// s3 handler don't respect flags
} catch (err) {
// retry without triggerring immutbaility check ,only on immutable remote
if (err.code === 'EPERM' && remoteIsImmutable) {
isImmutable = true
json = await this._handler.readFile(path, { flag: 'r' })
} else {
throw err
}
}
// _filename is a private field used to compute the backup id
//
// it's enumerable to make it cacheable
const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
// backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
if (typeof metadata.vm.is_a_template === 'number') {
@@ -827,29 +796,6 @@ export class RemoteAdapter {
}
return metadata
}
#computeTotalBackupSizeRecursively(backups) {
return reduce(
backups,
(prev, backup) => {
const _backup = Array.isArray(backup) ? this.#computeTotalBackupSizeRecursively(backup) : backup
return {
onDisk: prev.onDisk + (_backup.onDisk ?? _backup.size),
}
},
{ onDisk: 0 }
)
}
async getTotalVmBackupSize() {
return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups())
}
async getTotalBackupSize() {
const vmBackupSize = await this.getTotalVmBackupSize()
// @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented by fs
return vmBackupSize
}
}
Object.assign(RemoteAdapter.prototype, {

View File

@@ -21,7 +21,7 @@ export class RestoreMetadataBackup {
})
} else {
const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
const dataFileName = resolve(backupId, metadata.data ?? 'data.json')
const data = await handler.readFile(dataFileName)
// if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it

View File

@@ -2,10 +2,7 @@ import { createLogger } from '@xen-orchestra/log'
import { catchGlobalErrors } from '@xen-orchestra/log/configure'
import Disposable from 'promise-toolbox/Disposable'
import humanFormat from 'human-format'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import mapValues from 'lodash/mapValues.js'
import ms from 'ms'
import { compose } from '@vates/compose'
import { createCachedLookup } from '@vates/cached-dns.lookup'
import { createDebounceResource } from '@vates/disposable/debounceResource.js'
@@ -23,7 +20,7 @@ createCachedLookup().patchGlobal()
const logger = createLogger('xo:backups:worker')
catchGlobalErrors(logger)
const { debug, info, warn } = logger
const { debug } = logger
class BackupWorker {
#config
@@ -152,27 +149,6 @@ process.on('message', async message => {
debug('message received', { message })
if (message.action === 'run') {
const resourceStart = process.resourceUsage()
const timeStart = process.hrtime.bigint()
info('starting backup')
process.on('exit', exitCode => {
const resourceUsage = mapValues(process.resourceUsage(), (end, key) => end - resourceStart[key])
const cpuTotal = resourceUsage.userCPUTime + resourceUsage.systemCPUTime
const duration = Number((process.hrtime.bigint() - timeStart) / 1000n) // in μs
info('process will exit', {
duration,
exitCode,
resourceUsage,
summary: {
duration: ms(duration / 1000),
cpuUsage: Math.round((100 * cpuTotal) / duration) + '%',
memoryUsage: humanFormat.bytes(resourceUsage.maxRSS * 1024),
},
})
})
const backupWorker = new BackupWorker(message.data)
try {
const result = message.runWithLogs
@@ -201,16 +177,8 @@ process.on('message', async message => {
status: 'failure',
})
} finally {
info('backup has ended')
await ignoreErrors.call(backupWorker.debounceResource.flushAll())
process.disconnect()
setTimeout(() => {
warn('worker process did not exit automatically, forcing...')
// eslint-disable-next-line n/no-process-exit
process.exit()
}, 30e3).unref()
}
}
})

View File

@@ -3,7 +3,7 @@ import sum from 'lodash/sum.js'
import { asyncMap } from '@xen-orchestra/async-map'
import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
import { basename, dirname, resolve } from 'node:path'
import { dirname, resolve } from 'node:path'
import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
import { limitConcurrency } from 'limit-concurrency-decorator'
import { mergeVhdChain } from 'vhd-lib/merge.js'
@@ -179,8 +179,7 @@ export async function cleanVm(
vmDir,
{
fixMetadata,
remove = false,
removeTmp = remove,
remove,
merge,
mergeBlockConcurrency,
mergeLimiter = defaultMergeLimiter,
@@ -201,10 +200,6 @@ export async function cleanVm(
// remove broken VHDs
await asyncMap(vhds, async path => {
if(removeTmp && basename(path)[0] === '.'){
logInfo('deleting temporary VHD', { path })
return VhdAbstract.unlink(handler, path)
}
try {
await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
@@ -442,8 +437,7 @@ export async function cleanVm(
}
}
// no warning because a VHD can be unused for perfectly good reasons,
// e.g. the corresponding backup (metadata file) has been deleted
logWarn('unused VHD', { path: vhd })
if (remove) {
logInfo('deleting unused VHD', { path: vhd })
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))

View File

@@ -1,23 +1,52 @@
import groupBy from 'lodash/groupBy.js'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import omit from 'lodash/omit.js'
import { asyncMap } from '@xen-orchestra/async-map'
import { CancelToken } from 'promise-toolbox'
import { compareVersions } from 'compare-versions'
import { createVhdStreamWithLength } from 'vhd-lib'
import { defer } from 'golike-defer'
import { cancelableMap } from './_cancelableMap.mjs'
import { Task } from './Task.mjs'
import pick from 'lodash/pick.js'
import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
// in `other_config` of an incrementally replicated VM, contains the UUID of the source VM
export const TAG_BASE_DELTA = 'xo:base_delta'
// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
//
// added after the complete replication
export const TAG_BACKUP_SR = 'xo:backup:sr'
// in other_config of VDIs of an incrementally replicated VM, contains the UUID of the source VDI
export const TAG_COPY_SRC = 'xo:copy_of'
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
export async function exportIncrementalVm(
vm,
baseVdis = {},
{ cancelToken = CancelToken.none, nbdConcurrency = 1, preferNbd } = {}
baseVm,
{
cancelToken = CancelToken.none,
// Sets of UUIDs of VDIs that must be exported as full.
fullVdisRequired = new Set(),
disableBaseTags = false,
nbdConcurrency = 1,
preferNbd,
} = {}
) {
// refs of VM's VDIs → base's VDIs.
const baseVdis = {}
baseVm &&
baseVm.$VBDs.forEach(vbd => {
let vdi, snapshotOf
if ((vdi = vbd.$VDI) && (snapshotOf = vdi.$snapshot_of) && !fullVdisRequired.has(snapshotOf.uuid)) {
baseVdis[vdi.snapshot_of] = vdi
}
})
const streams = {}
const vdis = {}
@@ -38,16 +67,13 @@ export async function exportIncrementalVm(
}
// Look for a snapshot of this vdi in the base VM.
const baseVdi = baseVdis[vdi.$snapshot_of.uuid]
const baseVdi = baseVdis[vdi.snapshot_of]
vdis[vdiRef] = {
...vdi,
other_config: {
...vdi.other_config,
[BASE_DELTA_VDI]: baseVdi?.uuid,
[VM_UUID]:
vm.$snapshot_of?.uuid ?? // vm is a snapshot
vm.uuid, // vm is a not snapshot
[TAG_BASE_DELTA]: baseVdi && !disableBaseTags ? baseVdi.uuid : undefined,
},
$snapshot_of$uuid: vdi.$snapshot_of?.uuid,
$SR$uuid: vdi.$SR.uuid,
@@ -94,6 +120,13 @@ export async function exportIncrementalVm(
vifs,
vm: {
...vm,
other_config:
baseVm && !disableBaseTags
? {
...vm.other_config,
[TAG_BASE_DELTA]: baseVm.uuid,
}
: omit(vm.other_config, TAG_BASE_DELTA),
},
},
'streams',
@@ -172,7 +205,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
$defer.onFailure(() => newVdi.$destroy())
await newVdi.update_other_config(COPY_OF, vdi.uuid)
await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
if (vdi.virtual_size > newVdi.virtual_size) {
await newVdi.$callAsync('resize', vdi.virtual_size)
}
@@ -226,6 +259,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
if (typeof stream === 'function') {
stream = await stream()
}
if (stream.length === undefined) {
stream = await createVhdStreamWithLength(stream)
}
await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)

View File

@@ -1,165 +0,0 @@
import { formatDateTime } from '@xen-orchestra/xapi'
import assert from 'node:assert/strict'
// in `other_config` of an incrementally replicated VM or VDI
// contains the UUID of the object used as a base for an incremental export
// used to search for the replica of the base before applying a incremental replication
export const BASE_DELTA_VDI = 'xo:base_delta_vdi'
// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
//
// added after the complete replication
export const REPLICATED_TO_SR_UUID = 'xo:backup:sr'
// in other_config of VDIs of an incrementally replicated VM
// contains the UUID of the source exported object (snapshot or VM)
export const COPY_OF = 'xo:copy_of'
export const DATETIME = 'xo:backup:datetime'
export const JOB_ID = 'xo:backup:job'
export const SCHEDULE_ID = 'xo:backup:schedule'
// contains the number of delta in a chain, stored as a string
export const DELTA_CHAIN_LENGTH = 'xo:backup:deltaChainLength'
// contains the string true if this vdi has been exported successfully
export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
// the VM ( not the snapshot) uuid
export const VM_UUID = 'xo:backup:vm'
async function listVdiRefs(xapi, vmRef) {
return xapi.VM_getDisks(vmRef)
}
async function applyToVmAndVdis(xapi, vmRef, fn) {
const vdiRefs = await listVdiRefs(xapi, vmRef)
return Promise.all([fn('VM', vmRef), ...vdiRefs.map(vdiRef => fn('VDI', vdiRef))])
}
async function getDeltaChainLength(xapi, type, ref) {
const otherConfig = await xapi.getField(type, ref, 'other_config')
return Number(otherConfig[DELTA_CHAIN_LENGTH] ?? 0)
}
/**
* set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
* @param {Number} length
* @returns {Promise}
*/
export async function setVmDeltaChainLength(xapi, vmRef, length) {
return applyToVmAndVdis(xapi, vmRef, async (type, ref) => {
await xapi.setFieldEntry(type, ref, 'other_config', DELTA_CHAIN_LENGTH, String(length))
})
}
/**
* Compute the delta chain length of a VM and its associated VDIs
* if there is a discrependcy, use, the highest value
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
*/
export async function getVmDeltaChainLength(xapi, vmRef) {
const lengths = await applyToVmAndVdis(xapi, vmRef, async (type, ref) => getDeltaChainLength(xapi, type, ref))
return Math.max(...lengths)
}
/**
*
* Reset the other_config field of a VM and its VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
*/
export function resetVmOtherConfig(xapi, vmRef) {
return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
return xapi.setFieldEntries(type, ref, 'other_config', {
[DATETIME]: null,
[DELTA_CHAIN_LENGTH]: null,
[EXPORTED_SUCCESSFULLY]: null,
[JOB_ID]: null,
[SCHEDULE_ID]: null,
[VM_UUID]: null,
// REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
})
})
}
/**
*
* used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
*
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
*/
export async function populateVdisOtherConfig(xapi, vmRef) {
const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
const {
[DATETIME]: datetime,
[DELTA_CHAIN_LENGTH]: chainLength,
[EXPORTED_SUCCESSFULLY]: successfully,
[JOB_ID]: jobId,
[REPLICATED_TO_SR_UUID]: replicatedTo,
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
} = otherConfig
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
xapi.setFieldEntries(type, ref, 'other_config', {
[DATETIME]: datetime,
[DELTA_CHAIN_LENGTH]: chainLength,
[EXPORTED_SUCCESSFULLY]: successfully,
[JOB_ID]: jobId,
[REPLICATED_TO_SR_UUID]: replicatedTo,
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
})
)
}
/**
*
* set the other_config key related to a backup of a VM and its associated VDIs
*
* @param {Xapi} xapi
* @param {String} vmRef
* @param {*} settings
* @returns {PRomise}
*/
export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, scheduleId, vmUuid, srUuid = null, ...other }) {
assert.notEqual(timestamp, undefined)
assert.notEqual(jobId, undefined)
assert.notEqual(scheduleId, undefined)
assert.notEqual(vmUuid, undefined)
// srUuid is nullish for backup
assert.equal(Object.keys(other).length, 0)
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
xapi.setFieldEntries(type, ref, 'other_config', {
[REPLICATED_TO_SR_UUID]: srUuid,
[DATETIME]: formatDateTime(timestamp),
[JOB_ID]: jobId,
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
})
)
}
/**
*
* mark the export of he VM and its VDIs as successfull
*
* @param {Xapi} xapi
* @param {String} vmRef
* @returns {Promise}
*/
export async function markExportSuccessfull(xapi, vmRef) {
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
)
}

View File

@@ -6,12 +6,11 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
import { Task } from '../Task.mjs'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
import { runTask } from './_runTask.mjs'
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
const noop = Function.prototype
const DEFAULT_REMOTE_VM_SETTINGS = {
concurrency: 2,
copyRetention: 0,
@@ -19,10 +18,8 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
exportRetention: 0,
healthCheckSr: undefined,
healthCheckVmsWithTags: [],
healthCheckTimeout: '10m',
maxExportRate: 0,
maxMergedDeltasPerRun: Infinity,
nRetriesVmBackupFailures: 0,
timeout: 0,
validateVhdStreams: false,
vmTimeout: 0,
@@ -44,7 +41,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
await Disposable.use(
() => this._getAdapter(job.sourceRemote),
() => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -66,19 +62,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmsUuids)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = vmUuid => {
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
nTriesByVmId[vmUuid]++
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
const opts = {
baseSettings,
@@ -87,7 +72,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
healthCheckSr,
remoteAdapters,
schedule,
settings: vmSettings,
settings: { ...settings, ...allSettings[vmUuid] },
sourceRemoteAdapter,
throttleStream,
vmUuid,
@@ -101,46 +86,10 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
}
return sourceRemoteAdapter
.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
.then(vmBackups => {
// avoiding to create tasks for empty directories
if (vmBackups.length > 0) {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
return task
.run(async () => {
try {
const result = await vmBackup.run()
task.success(result)
return result
} catch (error) {
if (isLastRun) {
throw error
} else {
Task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: error.message,
})
queue.add(vmUuid)
}
}
})
.catch(noop)
}
})
return runTask(taskStart, () => vmBackup.run())
}
const { concurrency } = settings
const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
while (queue.size > 0) {
const vmIds = Array.from(queue)
queue.clear()
await asyncMapSettled(vmIds, _handleVm)
}
await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
}
)
}

View File

@@ -11,11 +11,8 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
import { FullXapi } from './_vmRunners/FullXapi.mjs'
const noop = Function.prototype
const DEFAULT_XAPI_VM_SETTINGS = {
bypassVdiChainsCheck: false,
cbtDestroySnapshotData: false,
checkpointSnapshot: false,
concurrency: 2,
copyRetention: 0,
@@ -25,10 +22,8 @@ const DEFAULT_XAPI_VM_SETTINGS = {
fullInterval: 0,
healthCheckSr: undefined,
healthCheckVmsWithTags: [],
healthCheckTimeout: '10m',
maxExportRate: 0,
maxMergedDeltasPerRun: Infinity,
nRetriesVmBackupFailures: 0,
offlineBackup: false,
offlineSnapshot: false,
snapshotRetention: 0,
@@ -58,7 +53,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const throttleStream = createStreamThrottle(settings.maxExportRate)
const config = this._config
await Disposable.use(
Disposable.all(
extractIdsFromSimplePattern(job.srs).map(id =>
@@ -95,98 +89,48 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmIds)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = vmUuid => {
const getVmTask = () => {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
return taskByVmId[vmUuid]
}
const vmBackupFailed = error => {
if (isLastRun) {
throw error
} else {
Task.warning(`Retry the VM backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: error.message,
})
queue.add(vmUuid)
}
}
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
nTriesByVmId[vmUuid]++
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
return this._getRecord('VM', vmUuid).then(
disposableVm =>
Disposable.use(disposableVm, async vm => {
if (taskStart.data.name_label === undefined) {
taskStart.data.name_label = vm.name_label
}
const task = getVmTask()
return task
.run(async () => {
const opts = {
baseSettings,
config,
getSnapshotNameLabel,
healthCheckSr,
job,
remoteAdapters,
schedule,
settings: vmSettings,
srs,
throttleStream,
vm,
}
let vmBackup
if (job.mode === 'delta') {
vmBackup = new IncrementalXapi(opts)
Disposable.use(disposableVm, vm => {
taskStart.data.name_label = vm.name_label
return runTask(taskStart, () => {
const opts = {
baseSettings,
config,
getSnapshotNameLabel,
healthCheckSr,
job,
remoteAdapters,
schedule,
settings: { ...settings, ...allSettings[vm.uuid] },
srs,
throttleStream,
vm,
}
let vmBackup
if (job.mode === 'delta') {
vmBackup = new IncrementalXapi(opts)
} else {
if (job.mode === 'full') {
vmBackup = new FullXapi(opts)
} else {
if (job.mode === 'full') {
vmBackup = new FullXapi(opts)
} else {
throw new Error(`Job mode ${job.mode} not implemented`)
}
throw new Error(`Job mode ${job.mode} not implemented`)
}
try {
const result = await vmBackup.run()
task.success(result)
return result
} catch (error) {
vmBackupFailed(error)
}
})
.catch(noop) // errors are handled by logs
}
return vmBackup.run()
})
}),
error =>
getVmTask().run(() => {
vmBackupFailed(error)
runTask(taskStart, () => {
throw error
})
)
}
const { concurrency } = settings
const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
while (queue.size > 0) {
const vmIds = Array.from(queue)
queue.clear()
await asyncMapSettled(vmIds, _handleVm)
}
await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
}
)
}

View File

@@ -31,7 +31,7 @@ export class PoolMetadataBackup {
const poolDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${schedule.id}/${pool.$id}`
const dir = `${poolDir}/${formatFilenameDate(timestamp)}`
const stream = (await this._exportPoolMetadata()).body
const stream = await this._exportPoolMetadata()
const fileName = `${dir}/data`
const metadata = JSON.stringify(

View File

@@ -29,10 +29,6 @@ export function forkStreamUnpipe(source) {
if (source.forks === 0) {
debug('no more forks, destroying original stream')
source.destroy(new Error('no more consumers for this stream'))
} else {
// a combination of stream.unpipe, onReadable and onData may stall stream here
// force it to flow again since we're piping it
source.resume()
}
})
return fork

View File

@@ -1,40 +1,52 @@
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { AbstractRemote } from './_AbstractRemote.mjs'
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
import { watchStreamSize } from '../../_watchStreamSize.mjs'
import { Task } from '../../Task.mjs'
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
_getRemoteWriter() {
return FullRemoteWriter
}
_filterTransferList(transferList) {
return transferList.filter(this._filterPredicate)
}
async _run() {
async _run($defer) {
const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
for (const metadata of transferList) {
const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
const sizeContainer = watchStreamSize(stream)
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
if (transferList.length > 0) {
for (const metadata of transferList) {
const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
const sizeContainer = watchStreamSize(stream)
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
await this._callWriters(
writer =>
writer.run({
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
streamLength: stream.length,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
sizeContainer,
}),
'writer.run()'
)
// for healthcheck
this._tags = metadata.vm.tags
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
await this._callWriters(
writer =>
writer.run({
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additionnal properties to it
streamLength: stream.length,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
sizeContainer,
}),
'writer.run()'
)
// for healthcheck
this._tags = metadata.vm.tags
}
} else {
Task.info('No new data to upload for this VM')
}
}
}
decorateMethodsWith(FullRemote, {
_run: defer,
})

View File

@@ -30,12 +30,10 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
const vm = this._vm
const exportedVm = this._exportedVm
const stream = this._throttleStream(
(
await this._xapi.VM_export(exportedVm.$ref, {
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
useSnapshot: false,
})
).body
await this._xapi.VM_export(exportedVm.$ref, {
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
useSnapshot: false,
})
)
const vdis = await exportedVm.$getDisks()

View File

@@ -1,109 +1,66 @@
import { asyncEach } from '@vates/async-each'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import assert from 'node:assert'
import * as UUID from 'uuid'
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
import mapValues from 'lodash/mapValues.js'
import { AbstractRemote } from './_AbstractRemote.mjs'
import { forkDeltaExport } from './_forkDeltaExport.mjs'
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
import { Disposable } from 'promise-toolbox'
import { openVhd } from 'vhd-lib'
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
import { Task } from '../../Task.mjs'
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
_getRemoteWriter() {
return IncrementalRemoteWriter
}
// we'll transfer the full list if at least one backup should be transfered
// to ensure we don't cut the delta chain
_filterTransferList(transferList) {
if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
return transferList
}
return []
}
async _selectBaseVm(metadata) {
// for each disk , get the parent
const baseUuidToSrcVdi = new Map()
// no previous backup for a base( =key) backup
if (metadata.isBase) {
return
}
await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
if (isDifferencing) {
const vmDir = getVmBackupDir(metadata.vm.uuid)
const path = `${vmDir}/${metadata.vhds[id]}`
// don't catch error : we can't recover if the source vhd are missing
await Disposable.use(openVhd(this._sourceRemoteAdapter._handler, path), vhd => {
baseUuidToSrcVdi.set(UUID.stringify(vhd.header.parentUuid), vdi.$snapshot_of$uuid)
})
}
})
const presentBaseVdis = new Map(baseUuidToSrcVdi)
await this._callWriters(
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
'writer.checkBaseVdis()',
false
)
// check if the parent vdi are present in all the remotes
baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
if (!presentBaseVdis.has(baseUuid)) {
throw new Error(`Missing vdi ${baseUuid} which is a base for a delta`)
}
})
// yeah , let's go
}
async _run() {
async _run($defer) {
const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
for (const metadata of transferList) {
assert.strictEqual(metadata.mode, 'delta')
await this._selectBaseVm(metadata)
await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
useChain: false,
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
const isVhdDifferencing = {}
if (transferList.length > 0) {
for (const metadata of transferList) {
assert.strictEqual(metadata.mode, 'delta')
await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
})
await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
useChain: false,
})
incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
await this._callWriters(
writer =>
writer.transfer({
deltaExport: forkDeltaExport(incrementalExport),
isVhdDifferencing,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
}),
'writer.transfer()'
)
// this will update parent name with the needed alias
await this._callWriters(
writer =>
writer.updateUuidAndChain({
isVhdDifferencing,
timestamp: metadata.timestamp,
vdis: incrementalExport.vdis,
}),
'writer.updateUuidAndChain()'
)
const isVhdDifferencing = {}
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
// for healthcheck
this._tags = metadata.vm.tags
await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
})
incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
await this._callWriters(
writer =>
writer.transfer({
deltaExport: forkDeltaExport(incrementalExport),
isVhdDifferencing,
timestamp: metadata.timestamp,
vm: metadata.vm,
vmSnapshot: metadata.vmSnapshot,
}),
'writer.transfer()'
)
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
// for healthcheck
this._tags = metadata.vm.tags
}
} else {
Task.info('No new data to upload for this VM')
}
}
}
export const IncrementalRemote = IncrementalRemoteVmBackupRunner
decorateMethodsWith(IncrementalRemoteVmBackupRunner, {
_run: defer,
})

View File

@@ -1,6 +1,8 @@
import { asyncEach } from '@vates/async-each'
import { asyncMap } from '@xen-orchestra/async-map'
import { createLogger } from '@xen-orchestra/log'
import { pipeline } from 'node:stream'
import findLast from 'lodash/findLast.js'
import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
import keyBy from 'lodash/keyBy.js'
import mapValues from 'lodash/mapValues.js'
@@ -13,15 +15,9 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
import { Task } from '../../Task.mjs'
import { watchStreamSize } from '../../_watchStreamSize.mjs'
import {
DATETIME,
DELTA_CHAIN_LENGTH,
EXPORTED_SUCCESSFULLY,
setVmDeltaChainLength,
markExportSuccessfull,
} from '../../_otherConfig.mjs'
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
const noop = Function.prototype
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
@@ -34,14 +30,17 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
}
async _copy() {
const baseVdis = this._baseVdis
const baseVm = this._baseVm
const vm = this._vm
const exportedVm = this._exportedVm
const fullVdisRequired = this._fullVdisRequired
const isFull = fullVdisRequired === undefined || fullVdisRequired.size !== 0
const isFull = Object.values(baseVdis).length === 0 || Object.values(baseVdis).some(_ => !_)
await this._callWriters(writer => writer.prepare({ isFull }), 'writer.prepare()')
const deltaExport = await exportIncrementalVm(exportedVm, baseVdis, {
const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
fullVdisRequired,
nbdConcurrency: this._settings.nbdConcurrency,
preferNbd: this._settings.preferNbd,
})
@@ -79,25 +78,18 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
'writer.transfer()'
)
// we want to control the uuid of the vhd in the chain
// and ensure they are correctly chained
await this._callWriters(
writer =>
writer.updateUuidAndChain({
isVhdDifferencing,
timestamp,
vdis: deltaExport.vdis,
}),
'writer.updateUuidAndChain()'
)
this._baseVm = exportedVm
if (!isFull) {
await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
} // on a full the delta chain will be null
if (baseVm !== undefined) {
await exportedVm.update_other_config(
'xo:backup:deltaChainLength',
String(+(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1)
)
}
// not the case if offlineBackup
if (exportedVm.is_a_snapshot) {
await markExportSuccessfull(this._xapi, exportedVm.$ref)
await exportedVm.update_other_config('xo:backup:exported', 'true')
}
const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
@@ -115,67 +107,43 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
async _selectBaseVm() {
const xapi = this._xapi
// filter _jobSnapshotVdis to have only the last successfully exported vdi
// compute delta chain length
// fill baseUuidToSrcVdi with this data
const exportedVdis = this._jobSnapshotVdis.filter(
_ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
)
let lastSuccessfullBackup
let lastExportedVdis = []
for (const exportedVdi of exportedVdis) {
if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
lastExportedVdis = []
lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
}
if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
lastExportedVdis.push(exportedVdi)
}
}
this._baseVdis = {}
if (lastExportedVdis.length === 0) {
debug('no base VDIS found', {
jobLength: this._jobSnapshotVdis.length,
lastSuccessfullBackup,
exportedLength: exportedVdis.length,
})
let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
if (baseVm === undefined) {
debug('no base VM found')
return
}
const deltaChainLength = Math.max(
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
)
const fullInterval = this._settings.fullInterval
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
debug('not using base VM because fullInterval reached', {
fullInterval,
deltaChainLength,
eq: fullInterval < deltaChainLength + 1,
dc1: deltaChainLength + 1,
})
const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
debug('not using base VM becaust fullInterval reached')
return
}
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
const baseUuidToSrcVdiUuid = new Map()
for (const lastExportedVdi of lastExportedVdis) {
const baseUuid = lastExportedVdi.uuid
const snapshotOf = lastExportedVdi.snapshot_of
// resolve full record
baseVm = await xapi.getRecord('VM', baseVm.$ref)
const baseUuidToSrcVdi = new Map()
await asyncMap(await baseVm.$getDisks(), async baseRef => {
const [baseUuid, snapshotOf] = await Promise.all([
xapi.getField('VDI', baseRef, 'uuid'),
xapi.getField('VDI', baseRef, 'snapshot_of'),
])
const srcVdi = srcVdis[snapshotOf]
if (srcVdi !== undefined) {
baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
baseUuidToSrcVdi.set(baseUuid, srcVdi)
} else {
debug('ignore snapshot VDI because no longer present on VM', {
vdi: baseUuid,
})
}
}
})
const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
const presentBaseVdis = new Map(baseUuidToSrcVdi)
await this._callWriters(
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseVm),
'writer.checkBaseVdis()',
false
)
@@ -184,22 +152,24 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
debug('no base VM found')
return
}
baseUuidToSrcVdiUuid.forEach((srcVdiUuid, baseUuid) => {
const fullVdisRequired = new Set()
baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
if (presentBaseVdis.has(baseUuid)) {
debug('found base VDI', {
base: baseUuid,
vdi: srcVdiUuid,
vdi: srcVdi.uuid,
})
this._baseVdis[srcVdiUuid] = lastExportedVdis.find(vdi => vdi.uuid === baseUuid)
} else {
debug('missing base VDI', {
base: baseUuid,
vdi: srcVdiUuid,
vdi: srcVdi.uuid,
})
// this will force a full for this vdi
this._baseVdis[srcVdiUuid] = undefined
fullVdisRequired.add(srcVdi.uuid)
}
})
this._deltaChainLength = deltaChainLength
this._baseVm = baseVm
this._fullVdisRequired = fullVdisRequired
}
}

View File

@@ -1,18 +1,12 @@
import groupBy from 'lodash/groupBy.js'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { asyncEach } from '@vates/async-each'
import { Disposable } from 'promise-toolbox'
import { createPredicate } from 'value-matcher'
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
import { Abstract } from './_Abstract.mjs'
import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
import { Task } from '../../Task.mjs'
export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
_filterPredicate
constructor({
config,
job,
@@ -28,7 +22,6 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this.config = config
this.job = job
this.remoteAdapters = remoteAdapters
this._settings = settings
this.scheduleId = schedule.id
this.timestamp = undefined
@@ -61,91 +54,46 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
})
)
})
const { filter } = job
if (filter === undefined) {
this._filterPredicate = () => true
} else {
this._filterPredicate = createPredicate(filter)
}
}
async #computeTransferListPerJob(sourceBackups, remotesBackups) {
async _computeTransferList(predicate) {
const vmBackups = await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, predicate)
const localMetada = new Map()
sourceBackups.forEach(metadata => {
Object.values(vmBackups).forEach(metadata => {
const timestamp = metadata.timestamp
localMetada.set(timestamp, metadata)
})
const nbRemotes = remotesBackups.length
const nbRemotes = Object.keys(this.remoteAdapters).length
const remoteMetadatas = {}
remotesBackups.forEach(async remoteBackups => {
remoteBackups.forEach(metadata => {
await asyncEach(Object.values(this.remoteAdapters), async remoteAdapter => {
const remoteMetadata = await remoteAdapter.listVmBackups(this._vmUuid, predicate)
remoteMetadata.forEach(metadata => {
const timestamp = metadata.timestamp
remoteMetadatas[timestamp] = (remoteMetadatas[timestamp] ?? 0) + 1
})
})
let transferList = []
let chain = []
const timestamps = [...localMetada.keys()]
timestamps.sort()
for (const timestamp of timestamps) {
if (remoteMetadatas[timestamp] !== nbRemotes) {
// this backup is not present in all the remote
// should be retransfered if not found later
transferList.push(localMetada.get(timestamp))
chain.push(localMetada.get(timestamp))
} else {
// backup is present in local and remote : the chain has already been transferred
transferList = []
chain = []
}
}
if (transferList.length > 0) {
const filteredTransferList = this._filterTransferList(transferList)
if (filteredTransferList.length > 0) {
return filteredTransferList
} else {
Task.info('This VM is excluded by the job filter')
return []
}
} else {
Task.info('No new data to upload for this VM')
}
return []
return chain
}
/**
*
* @param {*} vmPredicate a callback checking if backup is eligible for transfer. This filter MUST NOT cut delta chains
* @returns
*/
async _computeTransferList(vmPredicate) {
const sourceBackups = Object.values(await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
const remotesBackups = await Promise.all(
Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
)
const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
const transferByJobs = await Promise.all(
Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
)
)
return transferByJobs.flat(1)
}
async run($defer) {
async run() {
const handler = this._sourceRemoteAdapter._handler
await Disposable.use(await handler.lock(getVmBackupDir(this._vmUuid)), async () => {
await this._callWriters(async writer => {
await writer.beforeBackup()
$defer(async () => {
await writer.afterBackup()
})
}, 'writer.beforeBackup()')
await this._run()
await this._healthCheck()
})
}
}
decorateMethodsWith(AbstractRemote, {
run: defer,
})

View File

@@ -4,18 +4,11 @@ import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { asyncMap } from '@xen-orchestra/async-map'
import { decorateMethodsWith } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { formatDateTime } from '@xen-orchestra/xapi'
import { getOldEntries } from '../../_getOldEntries.mjs'
import { Task } from '../../Task.mjs'
import { Abstract } from './_Abstract.mjs'
import {
DATETIME,
JOB_ID,
SCHEDULE_ID,
populateVdisOtherConfig,
resetVmOtherConfig,
setVmOtherConfig,
} from '../../_otherConfig.mjs'
export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
constructor({
@@ -32,7 +25,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
vm,
}) {
super()
if (vm.other_config[JOB_ID] === job.id && 'start' in vm.blocked_operations) {
if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
// don't match replicated VMs created by this very job otherwise they
// will be replicated again and again
throw new Error('cannot backup a VM created by this very job')
@@ -56,17 +49,17 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
this._exportedVm = undefined
this._vm = vm
this._baseVdis = undefined
this._fullVdisRequired = undefined
this._getSnapshotNameLabel = getSnapshotNameLabel
this._isIncremental = job.mode === 'delta'
this._healthCheckSr = healthCheckSr
this._jobId = job.id
this._jobSnapshotVdis = undefined
this._jobSnapshots = undefined
this._throttleStream = throttleStream
this._xapi = vm.$xapi
// Base VM for the export
this._baseVdis = undefined
this._baseVm = undefined
// Settings for this specific run (job, schedule, VM)
if (tags.includes('xo-memory-backup')) {
@@ -130,8 +123,15 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
// copied on manual snapshots and interfere with the backup jobs
async _cleanMetadata() {
const vm = this._vm
if (JOB_ID in vm.other_config) {
await resetVmOtherConfig(this._xapi, vm.$ref)
if ('xo:backup:job' in vm.other_config) {
await vm.update_other_config({
'xo:backup:datetime': null,
'xo:backup:deltaChainLength': null,
'xo:backup:exported': null,
'xo:backup:job': null,
'xo:backup:schedule': null,
'xo:backup:vm': null,
})
}
}
@@ -146,29 +146,23 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
if (!settings.bypassVdiChainsCheck) {
await vm.$assertHealthyVdiChains()
}
if (settings.preferNbd) {
try {
// enable CBT on all disks if possible
const diskRefs = await xapi.VM_getDisks(vm.$ref)
await Promise.all(diskRefs.map(diskRef => xapi.call('VDI.enable_cbt', diskRef)))
} catch (error) {
Task.info(`couldn't enable CBT`, error)
}
}
const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
ignoredVdisTag: '[NOBAK]',
ignoreNobakVdis: true,
name_label: this._getSnapshotNameLabel(vm),
unplugVusbs: true,
})
this.timestamp = Date.now()
await setVmOtherConfig(xapi, snapshotRef, {
timestamp: this.timestamp,
jobId: this._jobId,
scheduleId: this.scheduleId,
vmUuid: vm.uuid,
await xapi.setFieldEntries('VM', snapshotRef, 'other_config', {
'xo:backup:datetime': formatDateTime(this.timestamp),
'xo:backup:job': this._jobId,
'xo:backup:schedule': this.scheduleId,
'xo:backup:vm': vm.uuid,
})
this._exportedVm = await xapi.getRecord('VM', snapshotRef)
return this._exportedVm.uuid
})
} else {
@@ -182,123 +176,40 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
const vmRef = this._vm.$ref
const xapi = this._xapi
// to ensure compatibility with snapshots older than CBT implementation
// update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
// remove by then end of 2024
const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
const snapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
const snapshotsOtherConfig = await asyncMap(snapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
const vmSnapshots = []
vmSnapshotsOtherConfig.forEach((other_config, i) => {
if (other_config[JOB_ID] === jobId) {
vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
const snapshots = []
snapshotsOtherConfig.forEach((other_config, i) => {
if (other_config['xo:backup:job'] === jobId) {
snapshots.push({ other_config, $ref: snapshotsRef[i] })
}
})
await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
// end of compatibiliy handling
// handle snapshot by VDI
this._jobSnapshotVdis = []
const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
for (const srcVdi of srcVdis) {
const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
for (const snapshot of snapshots) {
if (snapshot.other_config[JOB_ID] === jobId) {
this._jobSnapshotVdis.push(snapshot)
}
}
}
snapshots.sort((a, b) => (a.other_config['xo:backup:datetime'] < b.other_config['xo:backup:datetime'] ? -1 : 1))
this._jobSnapshots = snapshots
}
async _removeUnusedSnapshots() {
const allSettings = this.job.settings
const baseSettings = this._baseSettings
const baseVmRef = this._baseVm?.$ref
const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
const xapi = this._xapi
// ensure all the event has been processed by the xapi
await xapi.barrier()
// ensure cached object are up to date
this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
const datetimes = Object.keys(snapshotPerDatetime)
datetimes.sort()
await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
const settings = {
...baseSettings,
...allSettings[scheduleId],
...allSettings[this._vm.uuid],
}
// ensure we never delete the last one for delta
const minRetention = this.job.mode === 'delta' ? 1 : 0
const retention = Math.max(settings.snapshotRetention ?? 0, minRetention)
await asyncMap(getOldEntries(retention, datetimes), async datetime => {
const vdis = snapshotPerDatetime[datetime]
let vmRef
// if there is an attached VM => destroy the VM (Non CBT backups)
for (const vdi of vdis) {
const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
if (vbds.length > 0) {
// only one VM linked to this vdi
// this will throw error for VDI still attached to control domain
assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
const vm = vbds[0].$VM
assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
const vmRefVdi = vm.$ref
// same vm than other vdi of the same batch
assert.ok(
vmRef === undefined || vmRef === vmRefVdi,
'_removeUnusedSnapshots don t handle vdi related to multiple VMs '
)
vmRef = vmRefVdi
}
}
if (vmRef !== undefined) {
return xapi.VM_destroy(vmRef)
} else {
return asyncMap(
vdis.map(async ({ $ref }) => {
await xapi.VDI_destroy($ref)
})
)
return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
if ($ref !== baseVmRef) {
return xapi.VM_destroy($ref)
}
})
})
}
async _removeSnapshotData() {
// now that we use CBT, we can destroy the data of the snapshot used for this backup
// going back to a previous version of XO not supporting CBT will create a full backup
// this will only do something after snapshot and transfer
if (
// don't modify the VM
this._exportedVm?.is_a_snapshot &&
// user don't want to keep the snapshot data
this._settings.snapshotRetention === 0 &&
// preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
// in that case next runs will be full, but there is not an easy way to prevent that
this._settings.preferNbd &&
// only delete snapshost data if the config allows it
this._settings.cbtDestroySnapshotData
) {
Task.info('will delete snapshot data')
const vdiRefs = await this._xapi.VM_getDisks(this._exportedVm.$ref)
await this._xapi.call('VM.destroy', this._exportedVm.$ref)
for (const vdiRef of vdiRefs) {
try {
// data_destroy will fail with a VDI_NO_CBT_METADATA error if CBT is not enabled on this VDI
await this._xapi.VDI_dataDestroy(vdiRef)
Task.info(`Snapshot data has been deleted`, { vdiRef })
} catch (error) {
Task.warning(`Couldn't delete snapshot data`, { error, vdiRef })
}
}
}
}
async copy() {
throw new Error('Not implemented')
}
@@ -329,33 +240,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
})
}, 'writer.beforeBackup()')
const vm = this._vm
// block migration during the backup on the VM itself, not the latest snapshot
{
const { pool_migrate, migrate_send } = vm.blocked_operations
const reason = 'VM migration is blocked during backup'
await vm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
$defer(async () => {
// delete the entries if they did not exist previously or if they were
// equal to reason (which happen if a previous backup was interrupted
// before resetting them)
await vm.update_blocked_operations({
migrate_send: migrate_send === undefined || migrate_send === reason ? null : migrate_send,
pool_migrate: pool_migrate === undefined || pool_migrate === reason ? null : pool_migrate,
})
// 2024-08-19 - Work-around a XAPI bug where allowed_operations are not properly computed when blocked_operations is updated
//
// this is a problem because some clients (e.g. XenCenter) use this field to allow operations.
//
// internal source: https://team.vates.fr/vates/pl/mjmxnce9qfdx587r3qpe4z91ho
await vm.$call('update_allowed_operations')
})
}
await this._fetchJobSnapshots()
await this._selectBaseVm()
@@ -363,6 +247,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await this._cleanMetadata()
await this._removeUnusedSnapshots()
const vm = this._vm
const isRunning = vm.power_state === 'Running'
const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
if (startAfter) {
@@ -376,7 +261,15 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
}
if (this._writers.size !== 0) {
await this._copy()
const { pool_migrate = null, migrate_send = null } = this._exportedVm.blocked_operations
const reason = 'VM migration is blocked during backup'
await this._exportedVm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
try {
await this._copy()
} finally {
await this._exportedVm.update_blocked_operations({ pool_migrate, migrate_send })
}
}
} finally {
if (startAfter) {
@@ -385,7 +278,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await this._fetchJobSnapshots()
await this._removeUnusedSnapshots()
await this._removeSnapshotData()
}
await this._healthCheck()
}

View File

@@ -1,5 +1,6 @@
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import { formatDateTime } from '@xen-orchestra/xapi'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -8,7 +9,6 @@ import { Task } from '../../Task.mjs'
import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
import { listReplicatedVms } from './_listReplicatedVms.mjs'
import { setVmOtherConfig } from '../../_otherConfig.mjs'
export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
constructor(props) {
@@ -76,12 +76,14 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
'Start operation for this vm is blocked, clone it if you want to use it.'
)
),
setVmOtherConfig(xapi, targetVmRef, {
timestamp,
jobId: job.id,
scheduleId,
srUuid,
vmUuid: vm.uuid,
targetVm.update_other_config({
'xo:backup:sr': srUuid,
// these entries need to be added in case of offline backup
'xo:backup:datetime': formatDateTime(timestamp),
'xo:backup:job': job.id,
'xo:backup:schedule': scheduleId,
'xo:backup:vm': vm.uuid,
}),
])

View File

@@ -1,15 +1,17 @@
import assert from 'node:assert'
import mapValues from 'lodash/mapValues.js'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { asyncEach } from '@vates/async-each'
import { asyncMap } from '@xen-orchestra/async-map'
import { chainVhd, openVhd } from 'vhd-lib'
import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
import { createLogger } from '@xen-orchestra/log'
import { decorateClass } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { dirname, basename } from 'node:path'
import { dirname } from 'node:path'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
import { TAG_BASE_DELTA } from '../../_incrementalVm.mjs'
import { Task } from '../../Task.mjs'
import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
@@ -21,45 +23,42 @@ import { Disposable } from 'promise-toolbox'
const { warn } = createLogger('xo:backups:DeltaBackupWriter')
export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
#parentVdiPaths
#vhds
async checkBaseVdis(baseUuidToSrcVdi) {
this.#parentVdiPaths = {}
const { handler } = this._adapter
const adapter = this._adapter
const vdisDir = `${this._vmBackupDir}/vdis/${this._job.id}`
await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdiUuid]) => {
let parentDestPath
const vhdDir = `${vdisDir}/${srcVdiUuid}`
await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
let found = false
try {
const vhds = await handler.list(vhdDir, {
const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
ignoreMissing: true,
prependDir: true,
})
const packedBaseUuid = packUuid(baseUuid)
// the last one is probably the right one
for (let i = vhds.length - 1; i >= 0 && parentDestPath === undefined; i--) {
const path = vhds[i]
await asyncMap(vhds, async path => {
try {
if (await adapter.isMergeableParent(packedBaseUuid, path)) {
parentDestPath = path
}
await checkVhdChain(handler, path)
// Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
//
// since all the checks of a path are done in parallel, found would be containing
// only the last answer of isMergeableParent which is probably not the right one
// this led to the support tickets https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
found = found || isMergeable
} catch (error) {
warn('checkBaseVdis', { error })
await ignoreErrors.call(VhdAbstract.unlink(handler, path))
}
}
})
} catch (error) {
warn('checkBaseVdis', { error })
}
// no usable parent => the runner will have to decide to fall back to a full or stop backup
if (parentDestPath === undefined) {
if (!found) {
baseUuidToSrcVdi.delete(baseUuid)
} else {
this.#parentVdiPaths[vhdDir] = parentDestPath
}
})
}
@@ -124,45 +123,6 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
}
}
async updateUuidAndChain({ isVhdDifferencing, vdis }) {
assert.notStrictEqual(
this.#vhds,
undefined,
'_transfer must be called before updateUuidAndChain for incremental backups'
)
const parentVdiPaths = this.#parentVdiPaths
const { handler } = this._adapter
const vhds = this.#vhds
await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
const isDifferencing = isVhdDifferencing[`${id}.vhd`]
const path = `${this._vmBackupDir}/${vhds[id]}`
if (isDifferencing) {
assert.notStrictEqual(
parentVdiPaths,
undefined,
'checkbasevdi must be called before updateUuidAndChain for incremental backups'
)
const parentPath = parentVdiPaths[dirname(path)]
// we are in a incremental backup
// we already computed the chain in checkBaseVdis
assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
// forbid any kind of loop
assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
await chainVhd(handler, parentPath, handler, path)
}
// set the correct UUID in the VHD if needed
await Disposable.use(openVhd(handler, path), async vhd => {
if (!vhd.footer.uuid.equals(packUuid(vdi.uuid))) {
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
}
})
})
}
async _deleteOldEntries() {
const adapter = this._adapter
const oldEntries = this._oldEntries
@@ -181,10 +141,14 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
const jobId = job.id
const handler = adapter.handler
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// @todo : should skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
}
const basename = formatFilenameDate(timestamp)
// update this.#vhds before eventually skipping transfer, so that
// updateUuidAndChain has all the mandatory data
const vhds = (this.#vhds = mapValues(
const vhds = mapValues(
deltaExport.vdis,
vdi =>
`vdis/${jobId}/${
@@ -194,15 +158,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
vdi.uuid
: vdi.$snapshot_of$uuid
}/${adapter.getVhdFileName(basename)}`
))
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
Object.values(deltaExport.streams).forEach(stream => stream.destroy())
return { size: 0 }
}
)
metadataContent = {
isVhdDifferencing,
@@ -218,13 +174,38 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
vm,
vmSnapshot,
}
const { size } = await Task.run({ name: 'transfer' }, async () => {
let transferSize = 0
await asyncEach(
Object.keys(deltaExport.vdis),
async id => {
Object.entries(deltaExport.vdis),
async ([id, vdi]) => {
const path = `${this._vmBackupDir}/${vhds[id]}`
const isDifferencing = isVhdDifferencing[`${id}.vhd`]
let parentPath
if (isDifferencing) {
const vdiDir = dirname(path)
parentPath = (
await handler.list(vdiDir, {
filter: filename => filename[0] !== '.' && filename.endsWith('.vhd'),
prependDir: true,
})
)
.sort()
.pop()
assert.notStrictEqual(
parentPath,
undefined,
`missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config[TAG_BASE_DELTA]}`
)
parentPath = parentPath.slice(1) // remove leading slash
// TODO remove when this has been done before the export
await checkVhd(handler, parentPath)
}
// don't write it as transferSize += await async function
// since i += await asyncFun lead to race condition
// as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
@@ -236,6 +217,17 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
writeBlockConcurrency: this._config.writeBlockConcurrency,
})
transferSize += transferSizeOneDisk
if (isDifferencing) {
await chainVhd(handler, parentPath, handler, path)
}
// set the correct UUID in the VHD
await Disposable.use(openVhd(handler, path), async vhd => {
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
})
},
{
concurrency: settings.diskPerVmConcurrency,

View File

@@ -1,41 +1,42 @@
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import { formatDateTime } from '@xen-orchestra/xapi'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
import { importIncrementalVm } from '../../_incrementalVm.mjs'
import { importIncrementalVm, TAG_BACKUP_SR, TAG_BASE_DELTA, TAG_COPY_SRC } from '../../_incrementalVm.mjs'
import { Task } from '../../Task.mjs'
import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
import { listReplicatedVms } from './_listReplicatedVms.mjs'
import { COPY_OF, setVmOtherConfig, BASE_DELTA_VDI } from '../../_otherConfig.mjs'
import find from 'lodash/find.js'
import assert from 'node:assert'
export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
async checkBaseVdis(baseUuidToSrcVdi) {
async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
const sr = this._sr
const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(
vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
)
if (replicatedVm === undefined) {
return baseUuidToSrcVdi.clear()
}
// @todo use an index if possible
// @todo : this seems similare to decorateVmMetadata
const replicatedVdis = sr.$VDIs
.filter(vdi => {
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
return baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
const xapi = replicatedVm.$xapi
const replicatedVdis = new Set(
await asyncMap(await replicatedVm.$getDisks(), async vdiRef => {
const otherConfig = await xapi.getField('VDI', vdiRef, 'other_config')
return otherConfig[TAG_COPY_SRC]
})
.map(({ other_config }) => other_config?.[COPY_OF])
.filter(_ => !!_)
)
for (const uuid of baseUuidToSrcVdi.keys()) {
if (!replicatedVdis.includes(uuid)) {
if (!replicatedVdis.has(uuid)) {
baseUuidToSrcVdi.delete(uuid)
}
}
}
updateUuidAndChain() {
// nothing to do, the chaining is not modified in this case
}
prepare({ isFull }) {
// create the task related to this export and ensure all methods are called in this context
const task = new Task({
@@ -52,10 +53,10 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
this.cleanup = task.wrapFn(this.cleanup, !hasHealthCheckSr)
this.healthCheck = task.wrapFn(this.healthCheck, hasHealthCheckSr)
return task.run(() => this._prepare(isFull))
return task.run(() => this._prepare())
}
async _prepare(isFull) {
async _prepare() {
const settings = this._settings
const { uuid: srUuid, $xapi: xapi } = this._sr
const vmUuid = this._vmUuid
@@ -67,19 +68,14 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
if (settings.deleteFirst) {
// we want to keep the baseVM when copying a delta
// even if we want to keep only one after
let mostRecentEntry
if (this._oldEntries.length > 1 && settings.copyRetention === 1 && !isFull) {
mostRecentEntry = this._oldEntries.pop()
}
await this._deleteOldEntries()
this._oldEntries = mostRecentEntry !== undefined ? [mostRecentEntry] : []
}
}
async cleanup() {
await this._deleteOldEntries()
if (!this._settings.deleteFirst) {
await this._deleteOldEntries()
}
}
async _deleteOldEntries() {
@@ -89,39 +85,46 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
#decorateVmMetadata(backup) {
const { _warmMigration } = this._settings
const sr = this._sr
const xapi = sr.$xapi
const vm = backup.vm
vm.other_config[TAG_COPY_SRC] = vm.uuid
const remoteBaseVmUuid = vm.other_config[TAG_BASE_DELTA]
let baseVm
if (remoteBaseVmUuid) {
baseVm = find(
xapi.objects.all,
obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
)
vm.other_config[COPY_OF] = vm.uuid
if (!baseVm) {
throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
}
}
const baseVdis = {}
baseVm?.$VBDs.forEach(vbd => {
const vdi = vbd.$VDI
if (vdi !== undefined) {
baseVdis[vbd.VDI] = vbd.$VDI
}
})
vm.other_config[TAG_COPY_SRC] = vm.uuid
if (!_warmMigration) {
vm.tags.push('Continuous Replication')
}
// extracting the uuid of each delta vdi on the source
// get all in one pass, since there is a lot of objects
const sourceVdiUuids = Object.values(backup.vdis)
.map(({ other_config }) => other_config[BASE_DELTA_VDI])
// full vdi don't have a base
.filter(_ => !!_)
// @todo use index ?
const replicatedVdis = sr.$VDIs
.filter(vdi => {
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
})
Object.values(backup.vdis).forEach(vdi => {
vdi.other_config[COPY_OF] = vdi.uuid
if (sourceVdiUuids.length > 0) {
const baseReplicatedTo = replicatedVdis.find(
replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
)
assert.notStrictEqual(baseReplicatedTo, undefined)
vdi.baseVdi = baseReplicatedTo
} else {
vdi.baseVdi = undefined
}
// ensure the VDI are created on the target SR
vdi.other_config[TAG_COPY_SRC] = vdi.uuid
vdi.SR = sr.$ref
// vdi.other_config[TAG_BASE_DELTA] is never defined on a suspend vdi
if (vdi.other_config[TAG_BASE_DELTA]) {
const remoteBaseVdiUuid = vdi.other_config[TAG_BASE_DELTA]
const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
if (!baseVdi) {
throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
}
vdi.baseVdi = baseVdi
}
})
return backup
@@ -157,12 +160,14 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
'Start operation for this vm is blocked, clone it if you want to use it.'
)
),
setVmOtherConfig(xapi, targetVmRef, {
timestamp,
jobId: job.id,
scheduleId,
vmUuid: vm.uuid,
srUuid,
targetVm.update_other_config({
[TAG_BACKUP_SR]: srUuid,
// these entries need to be added in case of offline backup
'xo:backup:datetime': formatDateTime(timestamp),
'xo:backup:job': job.id,
'xo:backup:schedule': scheduleId,
[TAG_BASE_DELTA]: vm.uuid,
}),
])
}

View File

@@ -5,10 +5,6 @@ export class AbstractIncrementalWriter extends AbstractWriter {
throw new Error('Not implemented')
}
updateUuidAndChain() {
throw new Error('Not implemented')
}
cleanup() {
throw new Error('Not implemented')
}

View File

@@ -8,7 +8,6 @@ import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
import { ImportVmBackup } from '../../ImportVmBackup.mjs'
import { Task } from '../../Task.mjs'
import * as MergeWorker from '../../merge-worker/index.mjs'
import ms from 'ms'
const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
@@ -38,7 +37,6 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
},
lock: false,
mergeBlockConcurrency: this._config.mergeBlockConcurrency,
removeTmp: true,
})
})
} catch (error) {
@@ -79,11 +77,11 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
healthCheck() {
const sr = this._healthCheckSr
assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
if (this._metadataFileName === undefined) {
// this can happen when making a mirror backup with nothing to transfer
Task.info('no health check, since no backup have been transferred')
return
}
assert.notStrictEqual(
this._metadataFileName,
undefined,
'Metadata file name should be defined before making a health check'
)
return Task.run(
{
name: 'health check',
@@ -102,17 +100,10 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
additionnalVmTag: 'xo:no-bak=Health Check',
},
}).run()
let restoredVm
const restoredVm = xapi.getObject(restoredId)
try {
restoredVm = xapi.getObject(restoredId)
} catch (err) {
restoredVm = await xapi.waitObject(restoredId)
}
try {
const timeout = ms(this._settings.healthCheckTimeout)
await new HealthCheckVmBackup({
restoredVm,
timeout,
xapi,
}).run()
} finally {
@@ -122,13 +113,13 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
)
}
async _isAlreadyTransferred(timestamp) {
_isAlreadyTransferred(timestamp) {
const vmUuid = this._vmUuid
const adapter = this._adapter
const backupDir = getVmBackupDir(vmUuid)
try {
const actualMetadata = JSON.parse(
await adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
)
return actualMetadata
} catch (error) {}

View File

@@ -1,8 +1,8 @@
import { extractOpaqueRef } from '@xen-orchestra/xapi'
import assert from 'node:assert/strict'
import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
import { Task } from '../../Task.mjs'
import ms from 'ms'
export const MixinXapiWriter = (BaseClass = Object) =>
class MixinXapiWriter extends BaseClass {
@@ -26,8 +26,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
}
healthCheck() {
// the SR that the VM has been replicated on
const sr = this._sr
const sr = this._healthCheckSr
assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
// copy VM
@@ -39,36 +38,29 @@ export const MixinXapiWriter = (BaseClass = Object) =>
const { $xapi: xapi } = sr
let healthCheckVmRef
try {
let baseVm
try {
baseVm = xapi.getObject(this._targetVmRef)
} catch (err) {
baseVm = await xapi.waitObject(this._targetVmRef)
}
const baseVm = xapi.getObject(this._targetVmRef) ?? (await xapi.waitObject(this._targetVmRef))
if (await this.#isAlreadyOnHealthCheckSr(baseVm)) {
healthCheckVmRef = await Task.run(
{ name: 'cloning-vm' },
async () => await xapi.callAsync('VM.clone', this._targetVmRef, `Health Check - ${baseVm.name_label}`)
async () =>
await xapi
.callAsync('VM.clone', this._targetVmRef, `Health Check - ${baseVm.name_label}`)
.then(extractOpaqueRef)
)
} else {
healthCheckVmRef = await Task.run(
{ name: 'copying-vm' },
async () =>
await xapi.callAsync(
'VM.copy',
this._targetVmRef,
`Health Check - ${baseVm.name_label}`,
this._healthCheckSr.$ref
)
await xapi
.callAsync('VM.copy', this._targetVmRef, `Health Check - ${baseVm.name_label}`, sr.$ref)
.then(extractOpaqueRef)
)
}
const healthCheckVm =
xapi.getObject(healthCheckVmRef, undefined) ?? (await xapi.waitObject(healthCheckVmRef))
await healthCheckVm.add_tags('xo:no-bak=Health Check')
const timeout = ms(this._settings.healthCheckTimeout)
const healthCheckVm = xapi.getObject(healthCheckVmRef) ?? (await xapi.waitObject(healthCheckVmRef))
await healthCheckVm.add_tag('xo:no-bak=Health Check')
await new HealthCheckVmBackup({
restoredVm: healthCheckVm,
timeout,
xapi,
}).run()
} finally {

View File

@@ -1,7 +1,5 @@
import { DATETIME, JOB_ID, REPLICATED_TO_SR_UUID, SCHEDULE_ID, VM_UUID } from '../../_otherConfig.mjs'
const getReplicatedVmDatetime = vm => {
const { [DATETIME]: datetime = vm.name_label.slice(-17, -1) } = vm.other_config
const { 'xo:backup:datetime': datetime = vm.name_label.slice(-17, -1) } = vm.other_config
return datetime
}
@@ -18,11 +16,11 @@ export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
!object.is_a_snapshot &&
!object.is_a_template &&
'start' in object.blocked_operations &&
(oc[JOB_ID] === scheduleOrJobId || oc[SCHEDULE_ID] === scheduleOrJobId) &&
oc[REPLICATED_TO_SR_UUID] === srUuid &&
(oc[VM_UUID] === vmUuid ||
(oc['xo:backup:job'] === scheduleOrJobId || oc['xo:backup:schedule'] === scheduleOrJobId) &&
oc['xo:backup:sr'] === srUuid &&
(oc['xo:backup:vm'] === vmUuid ||
// 2018-03-28, JFT: to catch VMs replicated before this fix
oc[VM_UUID] === undefined)
oc['xo:backup:vm'] === undefined)
) {
vms[object.$id] = object
}

Some files were not shown because too many files have changed in this diff Show More