wip

Introduced by 7f4f86073

.eslintrc.js

@@ -15,9 +15,10 @@ module.exports = {
 
   overrides: [
     {
-      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
+      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js', '**/scripts/**.{,c,m}js'],
       rules: {
         'n/no-process-exit': 'off',
+        'n/shebang': 'off',
         'no-console': 'off',
       },
     },
@@ -46,6 +47,209 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ['@xen-orchestra/{web-core,lite,web}/**/*.{vue,ts}'],
+      parser: 'vue-eslint-parser',
+      parserOptions: {
+        sourceType: 'module',
+        parser: '@typescript-eslint/parser',
+      },
+      plugins: ['import'],
+      extends: [
+        'plugin:import/recommended',
+        'plugin:import/typescript',
+        'plugin:@typescript-eslint/recommended',
+        'plugin:vue/vue3-essential',
+      ],
+      settings: {
+        'import/resolver': {
+          typescript: true,
+          'eslint-import-resolver-custom-alias': {
+            alias: {
+              '@core': '../web-core/lib',
+              '@': './src',
+            },
+            extensions: ['.ts'],
+            packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
+          },
+        },
+      },
+      globals: {
+        XO_LITE_VERSION: true,
+        XO_LITE_GIT_HEAD: true,
+      },
+      rules: {
+        'import/order': [
+          'error',
+          {
+            groups: ['builtin', 'object', 'internal', ['type', 'external']],
+            alphabetize: {
+              order: 'asc',
+              caseInsensitive: true,
+            },
+            pathGroups: [
+              {
+                pattern: '@/**',
+                group: 'internal',
+              },
+              {
+                pattern: '@core/**',
+                group: 'internal',
+              },
+            ],
+          },
+        ],
+        'no-void': ['error', { allowAsStatement: true }],
+        'n/no-missing-import': 'off', // using 'import' plugin instead, to support TS aliases
+        'no-redeclare': 'off', // automatically checked by the TypeScript compiler
+        'no-dupe-class-members': 'off', // automatically checked by the TypeScript compiler
+        'no-undef': 'off', // automatically checked by the TypeScript compiler
+        '@typescript-eslint/no-explicit-any': 'off',
+        'vue/multi-word-component-names': 'off',
+        // Vue 3 - Strongly Recommended
+        'vue/attribute-hyphenation': 'error',
+        'vue/component-definition-name-casing': 'error',
+        'vue/first-attribute-linebreak': 'error',
+        'vue/html-closing-bracket-newline': 'off', // Conflicts with Prettier
+        'vue/html-closing-bracket-spacing': 'error',
+        'vue/html-end-tags': 'error',
+        'vue/html-indent': 'off', // Conflicts with Prettier
+        'vue/html-quotes': 'error',
+        'vue/html-self-closing': ['error', { html: { void: 'always', normal: 'always', component: 'always' } }],
+        'vue/max-attributes-per-line': 'off', // Conflicts with Prettier
+        'vue/multiline-html-element-content-newline': 'error',
+        'vue/mustache-interpolation-spacing': 'error',
+        'vue/no-multi-spaces': 'error',
+        'vue/no-spaces-around-equal-signs-in-attribute': 'error',
+        'vue/no-template-shadow': 'error',
+        'vue/prop-name-casing': 'error',
+        'vue/require-default-prop': 'off', // https://github.com/vuejs/eslint-plugin-vue/issues/2051
+        'vue/require-explicit-emits': 'error',
+        'vue/require-prop-types': 'error',
+        'vue/singleline-html-element-content-newline': 'off',
+        'vue/v-bind-style': 'error',
+        'vue/v-on-event-hyphenation': ['error', 'always', { autofix: true }],
+        'vue/v-on-style': 'error',
+        'vue/v-slot-style': 'error',
+        // Vue 3 - Recommended
+        'vue/attributes-order': 'error',
+        'vue/no-lone-template': 'error',
+        'vue/no-multiple-slot-args': 'error',
+        'vue/no-v-html': 'error',
+        'vue/this-in-template': 'error',
+        // Vue 3 - Uncategorized
+        'vue/block-order': ['error', { order: ['template', 'script', 'style'] }],
+        'vue/block-tag-newline': ['error', { singleline: 'consistent', multiline: 'always', maxEmptyLines: 0 }],
+        'vue/component-api-style': ['error', ['script-setup']],
+        'vue/component-name-in-template-casing': ['error', 'PascalCase', { registeredComponentsOnly: false }],
+        'vue/custom-event-name-casing': ['error', 'camelCase'],
+        'vue/define-emits-declaration': ['error', 'type-literal'],
+        'vue/define-macros-order': [
+          'error',
+          {
+            order: ['defineOptions', 'defineProps', 'defineEmits', 'defineModel', 'defineSlots'],
+            defineExposeLast: true,
+          },
+        ],
+        'vue/define-props-declaration': ['error', 'type-based'],
+        'vue/enforce-style-attribute': ['error', { allow: ['scoped', 'module'] }],
+        'vue/html-button-has-type': 'error',
+        'vue/html-comment-content-newline': ['error', { singleline: 'never', multiline: 'always' }],
+        'vue/html-comment-content-spacing': ['error', 'always'],
+        'vue/no-duplicate-attr-inheritance': 'error',
+        'vue/no-empty-component-block': 'error',
+        'vue/no-multiple-objects-in-class': 'error',
+        'vue/no-ref-object-reactivity-loss': 'error',
+        'vue/no-required-prop-with-default': 'error',
+        'vue/no-static-inline-styles': 'error',
+        'vue/no-template-target-blank': 'error',
+        'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
+        'vue/no-undef-properties': 'error',
+        'vue/no-unused-refs': 'error',
+        'vue/no-use-v-else-with-v-for': 'error',
+        'vue/no-useless-mustaches': 'error',
+        'vue/no-useless-v-bind': 'error',
+        'vue/no-v-text': 'error',
+        'vue/padding-line-between-blocks': 'error',
+        'vue/prefer-define-options': 'error',
+        'vue/prefer-separate-static-class': 'error',
+        'vue/prefer-true-attribute-shorthand': 'error',
+        'vue/require-macro-variable-name': 'error',
+        'vue/valid-define-options': 'error',
+      },
+    },
+    // Specific rules for XO i18n-related files
+    {
+      files: [
+        '@xen-orchestra/lite/**/*.{vue,ts,json}',
+        '@xen-orchestra/web/**/*.{vue,ts,json}',
+        '@xen-orchestra/web-core/**/*.{vue,ts,json}',
+      ],
+      extends: ['plugin:@intlify/vue-i18n/base'],
+      settings: {
+        'vue-i18n': {
+          localeDir: [
+            '@xen-orchestra/lite/src/locales/*.json',
+            '@xen-orchestra/web/src/locales/*.json',
+            '@xen-orchestra/web-core/lib/locales/*.json',
+          ],
+          messageSyntaxVersion: '^9.9.0',
+        },
+      },
+      rules: {
+        strict: 'off',
+        'no-irregular-whitespace': 'off',
+        '@intlify/vue-i18n/no-deprecated-i18n-component': 'error',
+        '@intlify/vue-i18n/no-deprecated-i18n-place-attr': 'error',
+        '@intlify/vue-i18n/no-deprecated-i18n-places-prop': 'error',
+        '@intlify/vue-i18n/no-deprecated-modulo-syntax': 'error',
+        '@intlify/vue-i18n/no-deprecated-tc': 'error',
+        '@intlify/vue-i18n/no-html-messages': 'error',
+        '@intlify/vue-i18n/no-i18n-t-path-prop': 'error',
+        '@intlify/vue-i18n/no-missing-keys': 'error',
+        '@intlify/vue-i18n/no-raw-text': [
+          'error',
+          {
+            ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
+          },
+        ],
+        '@intlify/vue-i18n/no-v-html': 'error',
+        '@intlify/vue-i18n/valid-message-syntax': 'error',
+        '@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
+        '@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa'] }],
+      },
+    },
+    // Specific rules for XO dev-related files
+    {
+      files: [
+        '@xen-orchestra/lite/src/stories/**/*',
+        '@xen-orchestra/lite/src/components/component-story/**/*',
+        '@xen-orchestra/web/src/pages/dev/**/*',
+        '@xen-orchestra/lite/src/views/story/**/*',
+      ],
+      rules: {
+        '@intlify/vue-i18n/no-raw-text': 'off',
+      },
+    },
+    {
+      files: ['@xen-orchestra/{web-core,lite,web}/src/pages/**/*.vue'],
+      parserOptions: {
+        sourceType: 'module',
+      },
+      rules: {
+        'vue/multi-word-component-names': 'off',
+      },
+    },
+    {
+      files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
+      parserOptions: {
+        sourceType: 'module',
+      },
+      rules: {
+        'eslint-comments/disable-enable-pair': 'off',
+        'eslint-comments/no-unlimited-disable': 'off',
+      },
+    },
   ],
 
   parserOptions: {
@@ -68,6 +272,11 @@ module.exports = {
 
     'no-console': ['error', { allow: ['warn', 'error'] }],
 
+    // this rule can prevent race condition bugs like parallel `a += await foo()`
+    //
+    // as it has a lots of false positive, it is only enabled as a warning for now
+    'require-atomic-updates': ['warn', { allowProperties: true }],
+
     strict: 'error',
   },
 }

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,48 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
-assignees: ''
----
-
-1. ⚠️ **If you don't follow this template, the issue will be closed**.
-2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
-
-Are you using XOA or XO from the sources?
-
-If XOA:
-
-- which release channel? (`stable` vs `latest`)
-- please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
-
-If XO from the sources:
-
-- Provide **your commit number**. If it's older than a week, we won't investigate
-- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
-- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Environment (please provide the following information):**
-
-- Node: [e.g. 16.12.1]
-- hypervisor: [e.g. XCP-ng 8.2.0]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,119 @@
+name: Bug Report
+description: Create a report to help us improve
+labels: ['type: bug :bug:', 'status: triaging :triangular_flag_on_post:']
+body:
+  - type: markdown
+    attributes:
+      value: |
+        1. ⚠️ **If you don't follow this template, the issue will be closed**.
+        2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
+  - type: markdown
+    attributes:
+      value: '## Are you using XOA or XO from the sources?'
+  - type: dropdown
+    id: xo-origin
+    attributes:
+      label: Are you using XOA or XO from the sources?
+      options:
+        - XOA
+        - XO from the sources
+        - both
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: '### If XOA:'
+  - type: dropdown
+    id: xoa-channel
+    attributes:
+      label: Which release channel?
+      description: please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
+      options:
+        - stable
+        - latest
+        - both
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: '### If XO from the sources:'
+  - type: markdown
+    attributes:
+      value: |
+        - Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
+        - As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
+  - type: input
+    id: xo-sources-commit-number
+    attributes:
+      label: Provide your commit number
+      description: If it's older than a week, we won't investigate
+      placeholder: e.g. 579f0
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: '## Bug description:'
+  - type: textarea
+    id: bug-description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: error-message
+    attributes:
+      label: Error message
+      render: Text
+    validations:
+      required: false
+  - type: textarea
+    id: steps
+    attributes:
+      label: To reproduce
+      description: 'Steps to reproduce the behavior:'
+      value: |
+        1. Go to '...'
+        2. Click on '...'
+        3. Scroll down to '...'
+        4. See error
+    validations:
+      required: false
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: false
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots
+      description: If applicable, add screenshots to help explain your problem
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: '## Environment (please provide the following information):'
+  - type: input
+    id: node-version
+    attributes:
+      label: Node
+      placeholder: e.g. 16.12.1
+    validations:
+      required: true
+  - type: input
+    id: hypervisor-version
+    attributes:
+      label: Hypervisor
+      placeholder: e.g. XCP-ng 8.2.0
+    validations:
+      required: true
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here
+    validations:
+      required: false

.github/workflows/ci.yml

@@ -24,8 +24,14 @@ jobs:
           cache: 'yarn'
       - name: Install project dependencies
         run: yarn
+      - name: Ensure packages are sorted in CHANGELOG.unreleased.md
+        run: ./scripts/gen-deps-list.js --check-order > /dev/null
+      - name: Ensure yarn.lock is up-to-date
+        run: git diff --exit-code yarn.lock
       - name: Build the project
         run: yarn build
+      - name: Unit tests
+        run: yarn test-unit
       - name: Lint tests
         run: yarn test-lint
       - name: Integration tests

.gitignore

@@ -30,8 +30,12 @@ pnpm-debug.log.*
 yarn-error.log
 yarn-error.log.*
 .env
+*.tsbuildinfo
 
 # code coverage
 .nyc_output/
 coverage/
 .turbo/
+
+# https://node-tap.org/dot-tap-folder/
+.tap/

.prettierignore

@@ -0,0 +1,6 @@
+# Ignore Handlebars files because partials are not currently supported
+# https://github.com/prettier/prettier/issues/11834
+**/*.hbs
+
+# Generated files
+/@xen-orchestra/web/typed-router.d.ts

@vates/async-each/package.json

@@ -33,8 +33,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^16.0.0",
-    "tap": "^16.3.0",
+    "sinon": "^18.0.0",
     "test": "^3.2.1"
   }
 }

@vates/decorate-with/.USAGE.md

@@ -62,6 +62,42 @@ decorateClass(Foo, {
 })
 ```
 
+### `decorateObject(object, map)`
+
+Decorates an object the same way `decorateClass()` decorates a class:
+
+```js
+import { decorateObject } from '@vates/decorate-with'
+
+const object = {
+  get bar() {
+    // body
+  },
+
+  set bar(value) {
+    // body
+  },
+
+  baz() {
+    // body
+  },
+}
+
+decorateObject(object, {
+  // getter and/or setter
+  bar: {
+    // without arguments
+    get: lodash.memoize,
+
+    // with arguments
+    set: [lodash.debounce, 150],
+  },
+
+  // method (with or without arguments)
+  baz: lodash.curry,
+})
+```
+
 ### `perInstance(fn, ...args)`
 
 Helper to decorate the method by instance instead of for the whole class.

@vates/decorate-with/README.md

@@ -80,6 +80,42 @@ decorateClass(Foo, {
 })
 ```
 
+### `decorateObject(object, map)`
+
+Decorates an object the same way `decorateClass()` decorates a class:
+
+```js
+import { decorateObject } from '@vates/decorate-with'
+
+const object = {
+  get bar() {
+    // body
+  },
+
+  set bar(value) {
+    // body
+  },
+
+  baz() {
+    // body
+  },
+}
+
+decorateObject(object, {
+  // getter and/or setter
+  bar: {
+    // without arguments
+    get: lodash.memoize,
+
+    // with arguments
+    set: [lodash.debounce, 150],
+  },
+
+  // method (with or without arguments)
+  baz: lodash.curry,
+})
+```
+
 ### `perInstance(fn, ...args)`
 
 Helper to decorate the method by instance instead of for the whole class.

@vates/decorate-with/index.js

@@ -14,10 +14,13 @@ function applyDecorator(decorator, value) {
 }
 
 exports.decorateClass = exports.decorateMethodsWith = function decorateClass(klass, map) {
-  const { prototype } = klass
+  return decorateObject(klass.prototype, map)
+}
+
+function decorateObject(object, map) {
   for (const name of Object.keys(map)) {
     const decorator = map[name]
-    const descriptor = getOwnPropertyDescriptor(prototype, name)
+    const descriptor = getOwnPropertyDescriptor(object, name)
     if (typeof decorator === 'function' || Array.isArray(decorator)) {
       descriptor.value = applyDecorator(decorator, descriptor.value)
     } else {
@@ -30,10 +33,11 @@ exports.decorateClass = exports.decorateMethodsWith = function decorateClass(kla
       }
     }
 
-    defineProperty(prototype, name, descriptor)
+    defineProperty(object, name, descriptor)
   }
-  return klass
+  return object
 }
+exports.decorateObject = decorateObject
 
 exports.perInstance = function perInstance(fn, decorator, ...args) {
   const map = new WeakMap()

@vates/decorate-with/package.json

@@ -20,7 +20,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/disposable/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "engines": {
     "node": ">=8.10"
   },
@@ -23,13 +23,13 @@
     "test": "node--test"
   },
   "dependencies": {
-    "@vates/multi-key-map": "^0.1.0",
+    "@vates/multi-key-map": "^0.2.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.6.0",
     "ensure-array": "^1.0.0"
   },
   "devDependencies": {
-    "sinon": "^16.0.0",
+    "sinon": "^18.0.0",
     "test": "^3.2.1"
   }
 }

@vates/event-listeners-manager/index.spec.js

@@ -20,6 +20,9 @@ function assertListeners(t, event, listeners) {
 }
 
 t.beforeEach(function (t) {
+  // work around https://github.com/tapjs/tapjs/issues/998
+  t.context = {}
+
   t.context.ee = new EventEmitter()
   t.context.em = new EventListenersManager(t.context.ee)
 })

@vates/event-listeners-manager/package.json

@@ -38,9 +38,9 @@
   "version": "1.0.1",
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "tap --branches=72"
+    "test": "tap --allow-incomplete-coverage"
   },
   "devDependencies": {
-    "tap": "^16.2.0"
+    "tap": "^18.7.0"
   }
 }

@vates/fuse-vhd/.USAGE.md

@@ -0,0 +1,28 @@
+Mount a vhd generated by xen-orchestra to filesystem
+
+### Library
+
+```js
+import { mount } from 'fuse-vhd'
+
+// return a disposable, see promise-toolbox/Disposable
+// unmount automatically when disposable is disposed
+// in case of differencing VHD, it mounts the full chain
+await mount(handler, diskId, mountPoint)
+```
+
+### cli
+
+From the install folder :
+
+```
+cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
+```
+
+After installing the package
+
+```
+xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
+```
+
+remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`

@vates/fuse-vhd/README.md

@@ -0,0 +1,59 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/fuse-vhd
+
+[![Package Version](https://badgen.net/npm/v/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd) ![License](https://badgen.net/npm/license/@vates/fuse-vhd) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/fuse-vhd)](https://bundlephobia.com/result?p=@vates/fuse-vhd) [![Node compatibility](https://badgen.net/npm/node/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/fuse-vhd):
+
+```sh
+npm install --save @vates/fuse-vhd
+```
+
+## Usage
+
+Mount a vhd generated by xen-orchestra to filesystem
+
+### Library
+
+```js
+import { mount } from 'fuse-vhd'
+
+// return a disposable, see promise-toolbox/Disposable
+// unmount automatically when disposable is disposed
+// in case of differencing VHD, it mounts the full chain
+await mount(handler, diskId, mountPoint)
+```
+
+### cli
+
+From the install folder :
+
+```
+cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
+```
+
+After installing the package
+
+```
+xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
+```
+
+remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/fuse-vhd/cli.mjs

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+import Disposable from 'promise-toolbox/Disposable'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+
+import { mount } from './index.mjs'
+
+async function* main([remoteUrl, vhdPathInRemote, mountPoint]) {
+  if (mountPoint === undefined) {
+    throw new TypeError('missing arg: cli <remoteUrl> <vhdPathInRemote> <mountPoint>')
+  }
+  const handler = yield getSyncedHandler({ url: remoteUrl })
+  const mounted = await mount(handler, vhdPathInRemote, mountPoint)
+
+  let disposePromise
+  process.on('SIGINT', async () => {
+    // ensure single dispose
+    if (!disposePromise) {
+      disposePromise = mounted.dispose()
+    }
+    await disposePromise
+    process.exit()
+  })
+}
+
+Disposable.wrap(main)(process.argv.slice(2))

@vates/fuse-vhd/index.mjs

@@ -1,8 +1,11 @@
 import LRU from 'lru-cache'
 import Fuse from 'fuse-native'
+import { createLogger } from '@xen-orchestra/log'
 import { VhdSynthetic } from 'vhd-lib'
 import { Disposable, fromCallback } from 'promise-toolbox'
 
+const { warn } = createLogger('vates:fuse-vhd')
+
 // build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
   mtime: st.mtime || new Date(),
@@ -52,13 +55,16 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
     },
     read(path, fd, buf, len, pos, cb) {
       if (path === '/vhd0') {
-        return vhd.readRawData(pos, len, cache, buf).then(cb)
+        return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
+          warn('read error', { path, len, pos, error })
+          cb(Fuse.EIO)
+        })
       }
       throw new Error(`read file ${path} not exists`)
     },
   })
   return new Disposable(
-    () => fromCallback(() => fuse.unmount()),
-    fromCallback(() => fuse.mount())
+    () => fromCallback(cb => fuse.unmount(cb)),
+    fromCallback(cb => fuse.mount(cb))
   )
 })

@vates/fuse-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/fuse-vhd",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "license": "ISC",
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,10 +19,15 @@
   },
   "main": "./index.mjs",
   "dependencies": {
+    "@xen-orchestra/fs": "^4.1.7",
+    "@xen-orchestra/log": "^0.6.0",
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.6.1"
+    "vhd-lib": "^4.11.0"
+  },
+  "bin": {
+    "xo-fuse-vhd": "./cli.mjs"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@vates/json-hash/.USAGE.md

@@ -0,0 +1,18 @@
+The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
+
+The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
+
+The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
+
+```js
+import { jsonHash } from '@vates/json-hash'
+
+console.log(jsonHash('foo'))
+// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
+
+// order of properties is ignored
+console.log(jsonHash({ foo: 0, bar: 1 }))
+// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
+console.log(jsonHash({ bar: 1, foo: 0 }))
+// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
+```

@vates/json-hash/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/json-hash/README.md

@@ -0,0 +1,51 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/json-hash
+
+[![Package Version](https://badgen.net/npm/v/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash) ![License](https://badgen.net/npm/license/@vates/json-hash) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/json-hash)](https://bundlephobia.com/result?p=@vates/json-hash) [![Node compatibility](https://badgen.net/npm/node/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash)
+
+> Compute a stable hash from a JSON-ifiable value
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/json-hash):
+
+```sh
+npm install --save @vates/json-hash
+```
+
+## Usage
+
+The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
+
+The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
+
+The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
+
+```js
+import { jsonHash } from '@vates/json-hash'
+
+console.log(jsonHash('foo'))
+// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
+
+// order of properties is ignored
+console.log(jsonHash({ foo: 0, bar: 1 }))
+// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
+console.log(jsonHash({ bar: 1, foo: 0 }))
+// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/json-hash/index.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const { createHash } = require('node:crypto')
+
+function updateJsonHash(value, hash) {
+  if (value !== null && typeof value === 'object') {
+    if (Array.isArray(value)) {
+      hash.update('[')
+      for (const item of value) {
+        updateJsonHash(item, hash)
+
+        // trailing is not a big deal because it does not need to be valid JSON
+        hash.update(',')
+      }
+      hash.update(']')
+    } else {
+      hash.update('{')
+      for (const key of Object.keys(value).sort()) {
+        updateJsonHash(key, hash)
+        hash.update(':')
+        updateJsonHash(value[key], hash)
+
+        // trailing is not a big deal because it does not need to be valid JSON
+        hash.update(',')
+      }
+      hash.update('}')
+    }
+  } else {
+    hash.update(JSON.stringify(value))
+  }
+}
+
+exports.jsonHash = function jsonHash(value) {
+  // this hash does not need to be secure, it just needs to be fast and with low collisions
+  const hash = createHash('sha256')
+  updateJsonHash(value, hash)
+  return hash.digest('base64url')
+}

@vates/json-hash/index.test.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const { jsonHash } = require('@vates/json-hash')
+const assert = require('node:assert/strict')
+const test = require('test')
+
+const cases = [
+  [null, 'dCNOmK_nSY-12vHzasLXiswzlGT5UHA7jAGYkvmCuQs'],
+  [42, 'c0dctApWjo2ooEXO0RATfhWfiQrE2og7axfcZRs6gEk'],
+  ['foo', 'siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs'],
+  [[], 'T1PNoYwrqgwDVLtfmj7L5e0Sq02OEbqHPC8RFhICuUU'],
+  [{}, 'RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o'],
+  [{ a: 1, b: 2 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
+  [{ b: 2, a: 1 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
+]
+
+for (const [value, hash] of cases) {
+  test(JSON.stringify(value), function () {
+    assert.equal(jsonHash(value), hash)
+  })
+}

@vates/json-hash/package.json

@@ -0,0 +1,33 @@
+{
+  "private": false,
+  "name": "@vates/json-hash",
+  "description": "Compute a stable hash from a JSON-ifiable value",
+  "keywords": [
+    "hash",
+    "json",
+    "stable"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/json-hash",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/json-hash",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.2.0",
+  "engines": {
+    "node": ">=14.18.0"
+  },
+  "devDependencies": {
+    "test": "^3.3.0"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "node--test"
+  }
+}

@vates/multi-key-map/.USAGE.md

@@ -17,4 +17,14 @@ map.get(['foo', 'bar']) // 2
 map.get(['bar', 'foo']) // 3
 map.get([OBJ]) // 4
 map.get([{}]) // undefined
+
+map.delete([])
+
+for (const [key, value] of map.entries() {
+  console.log(key, value)
+}
+
+for (const  value of map.values()) {
+  console.log(value)
+}
 ```

@vates/multi-key-map/README.md

@@ -35,6 +35,16 @@ map.get(['foo', 'bar']) // 2
 map.get(['bar', 'foo']) // 3
 map.get([OBJ]) // 4
 map.get([{}]) // undefined
+
+map.delete([])
+
+for (const [key, value] of map.entries() {
+  console.log(key, value)
+}
+
+for (const  value of map.values()) {
+  console.log(value)
+}
 ```
 
 ## Contributions

@vates/multi-key-map/index.js

@@ -36,14 +36,31 @@ function del(node, i, keys) {
   return node
 }
 
+function* entries(node, key) {
+  if (node !== undefined) {
+    if (node instanceof Node) {
+      const { value } = node
+      if (value !== undefined) {
+        yield [key, node.value]
+      }
+
+      for (const [childKey, child] of node.children.entries()) {
+        yield* entries(child, key.concat(childKey))
+      }
+    } else {
+      yield [key, node]
+    }
+  }
+}
+
 function get(node, i, keys) {
   return i === keys.length
     ? node instanceof Node
       ? node.value
       : node
     : node instanceof Node
-    ? get(node.children.get(keys[i]), i + 1, keys)
-    : undefined
+      ? get(node.children.get(keys[i]), i + 1, keys)
+      : undefined
 }
 
 function set(node, i, keys, value) {
@@ -69,6 +86,22 @@ function set(node, i, keys, value) {
   return node
 }
 
+function* values(node) {
+  if (node !== undefined) {
+    if (node instanceof Node) {
+      const { value } = node
+      if (value !== undefined) {
+        yield node.value
+      }
+      for (const child of node.children.values()) {
+        yield* values(child)
+      }
+    } else {
+      yield node
+    }
+  }
+}
+
 exports.MultiKeyMap = class MultiKeyMap {
   constructor() {
     // each node is either a value or a Node if it contains children
@@ -79,6 +112,10 @@ exports.MultiKeyMap = class MultiKeyMap {
     this._root = del(this._root, 0, keys)
   }
 
+  entries() {
+    return entries(this._root, [])
+  }
+
   get(keys) {
     return get(this._root, 0, keys)
   }
@@ -86,4 +123,8 @@ exports.MultiKeyMap = class MultiKeyMap {
   set(keys, value) {
     this._root = set(this._root, 0, keys, value)
   }
+
+  values() {
+    return values(this._root)
+  }
 }

@vates/multi-key-map/index.test.js

@@ -19,7 +19,7 @@ describe('MultiKeyMap', () => {
       // reverse composite key
       ['bar', 'foo'],
     ]
-    const values = keys.map(() => ({}))
+    const values = keys.map(() => Math.random())
 
     // set all values first to make sure they are all stored and not only the
     // last one
@@ -27,6 +27,12 @@ describe('MultiKeyMap', () => {
       map.set(key, values[i])
     })
 
+    assert.deepEqual(
+      Array.from(map.entries()),
+      keys.map((key, i) => [key, values[i]])
+    )
+    assert.deepEqual(Array.from(map.values()), values)
+
     keys.forEach((key, i) => {
       // copy the key to make sure the array itself is not the key
       assert.strictEqual(map.get(key.slice()), values[i])

@vates/multi-key-map/package.json

@@ -18,7 +18,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/nbd-client/index.mjs

@@ -1,10 +1,9 @@
 import assert from 'node:assert'
 import { Socket } from 'node:net'
 import { connect } from 'node:tls'
-import { fromCallback, pRetry, pDelay, pTimeout } from 'promise-toolbox'
+import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
 import { readChunkStrict } from '@vates/read-chunk'
 import { createLogger } from '@xen-orchestra/log'
-
 import {
   INIT_PASSWD,
   NBD_CMD_READ,
@@ -21,7 +20,6 @@ import {
   OPTS_MAGIC,
   NBD_CMD_DISC,
 } from './constants.mjs'
-
 const { warn } = createLogger('vates:nbd-client')
 
 // documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -36,10 +34,10 @@ export default class NbdClient {
   #exportSize
 
   #waitBeforeReconnect
-  #readAhead
   #readBlockRetries
   #reconnectRetry
   #connectTimeout
+  #messageTimeout
 
   // AFAIK, there is no guaranty the server answers in the same order as the queries
   // so we handle a backlog of command waiting for response and handle concurrency manually
@@ -52,17 +50,23 @@ export default class NbdClient {
   #reconnectingPromise
   constructor(
     { address, port = NBD_DEFAULT_PORT, exportname, cert },
-    { connectTimeout = 6e4, waitBeforeReconnect = 1e3, readAhead = 10, readBlockRetries = 5, reconnectRetry = 5 } = {}
+    {
+      connectTimeout = 6e4,
+      messageTimeout = 6e4,
+      waitBeforeReconnect = 1e3,
+      readBlockRetries = 5,
+      reconnectRetry = 5,
+    } = {}
   ) {
     this.#serverAddress = address
     this.#serverPort = port
     this.#exportName = exportname
     this.#serverCert = cert
     this.#waitBeforeReconnect = waitBeforeReconnect
-    this.#readAhead = readAhead
     this.#readBlockRetries = readBlockRetries
     this.#reconnectRetry = reconnectRetry
     this.#connectTimeout = connectTimeout
+    this.#messageTimeout = messageTimeout
   }
 
   get exportSize() {
@@ -115,13 +119,27 @@ export default class NbdClient {
     if (!this.#connected) {
       return
     }
+    this.#connected = false
+    const socket = this.#serverSocket
+
+    const queryId = this.#nextCommandQueryId
+    this.#nextCommandQueryId++
 
     const buffer = Buffer.alloc(28)
     buffer.writeInt32BE(NBD_REQUEST_MAGIC, 0) // it is a nbd request
     buffer.writeInt16BE(0, 4) // no command flags for a disconnect
     buffer.writeInt16BE(NBD_CMD_DISC, 6) // we want to disconnect from nbd server
-    await this.#write(buffer)
-    await this.#serverSocket.destroy()
+    buffer.writeBigUInt64BE(queryId, 8)
+    buffer.writeBigUInt64BE(0n, 16)
+    buffer.writeInt32BE(0, 24)
+    const promise = pFromCallback(cb => {
+      socket.end(buffer, 'utf8', cb)
+    })
+    try {
+      await pTimeout.call(promise, this.#messageTimeout)
+    } catch (error) {
+      socket.destroy()
+    }
     this.#serverSocket = undefined
     this.#connected = false
   }
@@ -195,11 +213,13 @@ export default class NbdClient {
   }
 
   #read(length) {
-    return readChunkStrict(this.#serverSocket, length)
+    const promise = readChunkStrict(this.#serverSocket, length)
+    return pTimeout.call(promise, this.#messageTimeout)
   }
 
   #write(buffer) {
-    return fromCallback.call(this.#serverSocket, 'write', buffer)
+    const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
+    return pTimeout.call(promise, this.#messageTimeout)
   }
 
   async #readInt32() {
@@ -232,19 +252,20 @@ export default class NbdClient {
     }
     try {
       this.#waitingForResponse = true
-      const magic = await this.#readInt32()
+      const buffer = await this.#read(16)
+      const magic = buffer.readInt32BE(0)
 
       if (magic !== NBD_REPLY_MAGIC) {
         throw new Error(`magic number for block answer is wrong : ${magic} ${NBD_REPLY_MAGIC}`)
       }
 
-      const error = await this.#readInt32()
+      const error = buffer.readInt32BE(4)
       if (error !== 0) {
         // @todo use error code from constants.mjs
         throw new Error(`GOT ERROR CODE  : ${error}`)
       }
 
-      const blockQueryId = await this.#readInt64()
+      const blockQueryId = buffer.readBigUInt64BE(8)
       const query = this.#commandQueryBacklog.get(blockQueryId)
       if (!query) {
         throw new Error(` no query associated with id ${blockQueryId}`)
@@ -265,7 +286,7 @@ export default class NbdClient {
     }
   }
 
-  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
+  async #readBlock(index, size) {
     // we don't want to add anything in backlog while reconnecting
     if (this.#reconnectingPromise) {
       await this.#reconnectingPromise
@@ -281,7 +302,13 @@ export default class NbdClient {
     buffer.writeInt16BE(NBD_CMD_READ, 6) // we want to read a data block
     buffer.writeBigUInt64BE(queryId, 8)
     // byte offset in the raw disk
-    buffer.writeBigUInt64BE(BigInt(index) * BigInt(size), 16)
+    const offset = BigInt(index) * BigInt(size)
+    const remaining = this.#exportSize - offset
+    if (remaining < BigInt(size)) {
+      size = Number(remaining)
+    }
+
+    buffer.writeBigUInt64BE(offset, 16)
     buffer.writeInt32BE(size, 24)
 
     return new Promise((resolve, reject) => {
@@ -307,45 +334,13 @@ export default class NbdClient {
     })
   }
 
-  async *readBlocks(indexGenerator) {
-    // default : read all blocks
-    if (indexGenerator === undefined) {
-      const exportSize = this.#exportSize
-      const chunkSize = 2 * 1024 * 1024
-      indexGenerator = function* () {
-        const nbBlocks = Math.ceil(Number(exportSize / BigInt(chunkSize)))
-        for (let index = 0; BigInt(index) < nbBlocks; index++) {
-          yield { index, size: chunkSize }
-        }
-      }
-    }
-    const readAhead = []
-    const readAheadMaxLength = this.#readAhead
-    const makeReadBlockPromise = (index, size) => {
-      const promise = pRetry(() => this.readBlock(index, size), {
-        tries: this.#readBlockRetries,
-        onRetry: async err => {
-          warn('will retry reading block ', index, err)
-          await this.reconnect()
-        },
-      })
-      // error is handled during unshift
-      promise.catch(() => {})
-      return promise
-    }
-
-    // read all blocks, but try to keep readAheadMaxLength promise waiting ahead
-    for (const { index, size } of indexGenerator()) {
-      // stack readAheadMaxLength promises before starting to handle the results
-      if (readAhead.length === readAheadMaxLength) {
-        // any error will stop reading blocks
-        yield readAhead.shift()
-      }
-
-      readAhead.push(makeReadBlockPromise(index, size))
-    }
-    while (readAhead.length > 0) {
-      yield readAhead.shift()
-    }
+  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
+    return pRetry(() => this.#readBlock(index, size), {
+      tries: this.#readBlockRetries,
+      onRetry: async err => {
+        warn('will retry reading block ', index, err)
+        await this.reconnect()
+      },
+    })
   }
 }

@vates/nbd-client/multi.mjs

@@ -0,0 +1,116 @@
+import { asyncEach } from '@vates/async-each'
+import { NBD_DEFAULT_BLOCK_SIZE } from './constants.mjs'
+import NbdClient from './index.mjs'
+import { createLogger } from '@xen-orchestra/log'
+
+const { warn } = createLogger('vates:nbd-client:multi')
+export default class MultiNbdClient {
+  #clients = []
+  #nbdConcurrency
+  #options
+  #readAhead
+  #settings
+
+  get exportSize() {
+    return this.#clients[0].exportSize
+  }
+
+  constructor(settings, { nbdConcurrency = 8, readAhead = 16, ...options } = {}) {
+    this.#readAhead = readAhead
+    this.#options = options
+    this.#nbdConcurrency = nbdConcurrency
+    if (!Array.isArray(settings)) {
+      settings = [settings]
+    }
+    this.#settings = settings
+  }
+
+  /**
+   *
+   * open nbdConcurrency connections to NBD servers
+   * it must obtain at least one connection to succeed
+   * it tries to spread connections on multiple host
+   */
+  async connect() {
+    const candidates = [...this.#settings]
+
+    const promises = []
+    const baseOptions = this.#options
+    const  _connect = async () => {
+      if (candidates.length === 0) {
+        return
+      }
+      // a little bit of randomization to spread the load
+      const nbdInfo = candidates[Math.floor(Math.random() * candidates.length)]
+      const client = new NbdClient(nbdInfo, {
+        ...baseOptions,
+        readAhead: Math.ceil(this.#readAhead / this.#nbdConcurrency),
+      })
+      try {
+        await client.connect()
+        this.#clients.push(client)
+      } catch (err) {
+        client.disconnect().catch(() => {})
+        // do not hammer unreachable hosts, once failed, remove from the list
+        const candidateIndex = candidates.findIndex(({ address }) => address === nbdInfo.address)
+        if (candidateIndex >= 0) {
+          // this candidate may have already been deleted by another parallel promise
+          candidates.splice(candidateIndex, 1)
+        }
+
+        warn(`can't connect to one nbd client`, { err })
+        // retry with another candidate (if available)
+        return _connect()
+      }
+    }
+    for (let i = 0; i < this.#nbdConcurrency; i++) {
+      promises.push(_connect())
+    }
+    await Promise.all(promises)
+
+    if (this.#clients.length === 0) {
+      throw new Error(`Fail to connect to any Nbd client`)
+    }
+    if (this.#clients.length < this.#nbdConcurrency) {
+      warn(
+        `incomplete connection by multi Nbd, only ${this.#clients.length} over ${this.#nbdConcurrency} expected clients`
+      )
+    }
+  }
+
+  async disconnect() {
+    await asyncEach(this.#clients, client => client.disconnect(), {
+      stopOnError: false,
+    })
+  }
+
+  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
+    const clientId = index % this.#clients.length
+    return this.#clients[clientId].readBlock(index, size)
+  }
+
+  async *readBlocks(indexGenerator) {
+    // default : read all blocks
+    const readAhead = []
+    const makeReadBlockPromise = (index, size) => {
+      const promise = this.readBlock(index, size)
+      // error is handled during unshift
+      promise.catch(() => {})
+      return promise
+    }
+
+    // read all blocks, but try to keep readAheadMaxLength promise waiting ahead
+    for (const { index, size } of indexGenerator()) {
+      // stack readAheadMaxLength promises before starting to handle the results
+      if (readAhead.length === this.#readAhead) {
+        // any error will stop reading blocks
+        yield readAhead.shift()
+      }
+
+      readAhead.push(makeReadBlockPromise(index, size))
+    }
+    while (readAhead.length > 0) {
+      yield readAhead.shift()
+    }
+  }
+}

@vates/nbd-client/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.0.0",
+  "version": "3.1.0",
   "engines": {
     "node": ">=14.0"
   },
@@ -24,14 +24,14 @@
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.6.0",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^1.3.6"
+    "xen-api": "^4.2.0"
   },
   "devDependencies": {
-    "tap": "^16.3.0",
+    "tap": "^18.7.0",
     "tmp": "^0.2.1"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
+    "test-integration": "tap --allow-incomplete-coverage"
   }
 }

@vates/nbd-client/tests/nbdclient.integ.mjs

@@ -1,4 +1,3 @@
-import NbdClient from '../index.mjs'
 import { spawn, exec } from 'node:child_process'
 import fs from 'node:fs/promises'
 import { test } from 'tap'
@@ -7,8 +6,10 @@ import { pFromCallback } from 'promise-toolbox'
 import { Socket } from 'node:net'
 import { NBD_DEFAULT_PORT } from '../constants.mjs'
 import assert from 'node:assert'
+import MultiNbdClient from '../multi.mjs'
 
-const FILE_SIZE = 10 * 1024 * 1024
+const CHUNK_SIZE = 1024 * 1024 // non default size
+const FILE_SIZE = 1024 * 1024 * 9.5 // non aligned file size
 
 async function createTempFile(size) {
   const tmpPath = await pFromCallback(cb => tmp.file(cb))
@@ -77,15 +78,14 @@ async function killNbdKit() {
   )
 }
 
-test('it works with unsecured network', async tap => {
+test('it works secured network', async tap => {
   const path = await createTempFile(FILE_SIZE)
 
   let nbdServer = await spawnNbdKit(path)
-  const client = new NbdClient(
-    {
-      address: '127.0.0.1',
-      exportname: 'MY_SECRET_EXPORT',
-      cert: `-----BEGIN CERTIFICATE-----
+  const connectionSettings = {
+    address: '127.0.0.1',
+    exportname: 'MY_SECRET_EXPORT',
+    cert: `-----BEGIN CERTIFICATE-----
 MIIDazCCAlOgAwIBAgIUeHpQ0IeD6BmP2zgsv3LV3J4BI/EwDQYJKoZIhvcNAQEL
 BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
 GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA1MTcxMzU1MzBaFw0yNDA1
@@ -107,15 +107,21 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
 57ABhfYpMlZ4aHjuN1bL
 -----END CERTIFICATE-----
 `,
-    },
-    {
-      readAhead: 2,
-    }
-  )
+  }
+  const invalid = {
+    address: '500.500.500.500',
+    port: 0,
+    exportname: 'nop',
+  }
+  // it should work even with some broken servers
+  const nbdInfos = [connectionSettings, connectionSettings, invalid]
+  const client = new MultiNbdClient(nbdInfos, {
+    nbdConcurrency: 4,
+    readAhead: 2,
+  })
 
   await client.connect()
   tap.equal(client.exportSize, BigInt(FILE_SIZE))
-  const CHUNK_SIZE = 1024 * 1024 // non default size
   const indexes = []
   for (let i = 0; i < FILE_SIZE / CHUNK_SIZE; i++) {
     indexes.push(i)
@@ -127,9 +133,9 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
   })
   let i = 0
   for await (const block of nbdIterator) {
-    let blockOk = true
+    let blockOk = block.length === Math.min(CHUNK_SIZE, FILE_SIZE - CHUNK_SIZE * i)
     let firstFail
-    for (let j = 0; j < CHUNK_SIZE; j += 4) {
+    for (let j = 0; j < block.length; j += 4) {
       const wanted = i * CHUNK_SIZE + j
       const found = block.readUInt32BE(j)
       blockOk = blockOk && found === wanted
@@ -137,7 +143,7 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
         firstFail = j
       }
     }
-    tap.ok(blockOk, `check block ${i} content`)
+    tap.ok(blockOk, `check block ${i} content ${block.length}`)
     i++
 
     // flaky server is flaky
@@ -147,17 +153,6 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
       nbdServer = await spawnNbdKit(path)
     }
   }
-
-  // we can reuse the conneciton to read other blocks
-  // default iterator
-  const nbdIteratorWithDefaultBlockIterator = client.readBlocks()
-  let nb = 0
-  for await (const block of nbdIteratorWithDefaultBlockIterator) {
-    nb++
-    tap.equal(block.length, 2 * 1024 * 1024)
-  }
-
-  tap.equal(nb, 5)
   assert.rejects(() => client.readBlock(100, CHUNK_SIZE))
 
   await client.disconnect()
@@ -166,3 +161,18 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
   nbdServer.kill()
   await fs.unlink(path)
 })
+
+test('fails if server does not answer', async () => {
+  const client = new MultiNbdClient(
+    {
+      address: '500.500.500.500',
+      port: 0,
+      exportname: 'nop',
+    },
+    {
+      nbdConcurrency: 1,
+      readAhead: 2,
+    }
+  )
+  await assert.rejects(client.connect())
+})

@vates/node-vsphere-soap/lib/client.mjs

@@ -17,7 +17,20 @@ import https from 'node:https'
 import util from 'util'
 import soap from 'soap'
 import Cookie from 'soap-cookie' // required for session persistence
+import { createLogger } from '@xen-orchestra/log'
 
+const { warn } = createLogger('xo:node-vsphere-soap:client')
+class VmwareError extends Error {
+  constructor(rawError) {
+    super(rawError.body)
+    const matches = rawError.body?.match(/<faultstring[^>]*>(.*)<\/faultstring>/im)
+    this.message = matches?.[1] ?? rawError.message
+    this.name = 'VmwareError'
+    this.stack = rawError.stack
+    // not putting the cause since the error failed to be stringified correctly down the road
+    warn(rawError)
+  }
+}
 // Client class
 // inherits from EventEmitter
 // possible events: connect, error, ready
@@ -31,14 +44,20 @@ export function Client(vCenterHostname, username, password, sslVerify) {
   EventEmitter.call(this)
 
   // sslVerify argument handling
+  // don't use any proxy to connect to the esxi
   if (sslVerify) {
-    this.clientopts = {}
+    this.clientopts = {
+      request: axios.create({
+        proxy: false,
+      }),
+    }
   } else {
     this.clientopts = {
       request: axios.create({
         httpsAgent: new https.Agent({
           rejectUnauthorized: false,
         }),
+        proxy: false,
       }),
     }
   }
@@ -134,8 +153,9 @@ Client.prototype._connect = function () {
     self.clientopts,
     function (err, client) {
       if (err) {
-        self.emit('error', err)
-        throw err
+        const vErr = new VmwareError(err)
+        self.emit('error', vErr)
+        throw vErr
       }
 
       self.client = client // save client for later use
@@ -144,6 +164,7 @@ Client.prototype._connect = function () {
         .runCommand('RetrieveServiceContent', { _this: 'ServiceInstance' })
         .once('result', function (result, raw, soapHeader) {
           if (!result.returnval) {
+            // this is a domain/server error return it as is
             self.status = 'disconnected'
             self.emit('error', raw)
             return
@@ -202,7 +223,7 @@ Client.prototype._close = function () {
 function _soapErrorHandler(self, emitter, command, args, err) {
   err = err || { body: 'general error' }
 
-  if (err.body.match(/session is not authenticated/)) {
+  if (err.body?.match(/session is not authenticated/)) {
     self.status = 'disconnected'
     process.removeAllListeners('beforeExit')
 
@@ -214,16 +235,19 @@ function _soapErrorHandler(self, emitter, command, args, err) {
           emitter.emit('result', result, raw, soapHeader)
         })
         .once('error', function (err) {
-          emitter.emit('error', err.body)
-          throw err
+          const vErr = new VmwareError(err)
+          emitter.emit('error', vErr)
+          throw vErr
         })
     } else {
-      emitter.emit('error', err.body)
-      throw err
+      const vErr = new VmwareError(err)
+      emitter.emit('error', vErr)
+      throw vErr
     }
   } else {
-    emitter.emit('error', err.body)
-    throw err
+    const vErr = new VmwareError(err)
+    emitter.emit('error', vErr)
+    throw vErr
   }
 }
 

@vates/node-vsphere-soap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/node-vsphere-soap",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
   "main": "lib/client.mjs",
   "author": "reedog117",
@@ -11,6 +11,7 @@
   },
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
+    "@xen-orchestra/log": "^0.6.0",
     "axios": "^1.4.0",
     "soap": "^1.0.0",
     "soap-cookie": "^0.10.1"

@vates/obfuscate/.USAGE.md

@@ -0,0 +1,46 @@
+### `replace(value, replacement)`
+
+> Recursively replace sensitive entries with a replacement value.
+
+```js
+import { replace } from '@vates/obfuscate'
+
+replace({ username: 'foo', password: 'bar' }, 'obfuscated')
+// → { username: 'foo', password: 'obfuscated'}
+```
+
+### `obfuscate(value)`
+
+> Recursively replace sensitive entries with a special tag.
+
+```js
+import { obfuscate } from '@vates/obfuscate'
+
+obfuscate({ username: 'foo', password: 'bar' })
+// → {
+//   username: 'foo',
+//   password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
+// }
+```
+
+### `merge(newValue, oldValue)`
+
+> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
+>
+> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
+
+```js
+import { obfuscate, merge } from '@vates/obfuscate'
+
+const original = { username: 'foo', password: 'bar', token: 'baz' }
+
+// compute an obfuscated value to send to a client
+const obfuscated = obfuscate(original)
+
+// let the client update any entries
+obfuscated.token = 'qux'
+
+// merge the updated value with the original one
+merge(obfuscated, original)
+// → { username: 'foo', password: 'bar', token: 'qux' }
+```

@vates/obfuscate/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/obfuscate/README.md

@@ -0,0 +1,77 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/obfuscate
+
+[![Package Version](https://badgen.net/npm/v/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate) ![License](https://badgen.net/npm/license/@vates/obfuscate) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/obfuscate)](https://bundlephobia.com/result?p=@vates/obfuscate) [![Node compatibility](https://badgen.net/npm/node/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/obfuscate):
+
+```sh
+npm install --save @vates/obfuscate
+```
+
+## Usage
+
+### `replace(value, replacement)`
+
+> Recursively replace sensitive entries with a replacement value.
+
+```js
+import { replace } from '@vates/obfuscate'
+
+replace({ username: 'foo', password: 'bar' }, 'obfuscated')
+// → { username: 'foo', password: 'obfuscated'}
+```
+
+### `obfuscate(value)`
+
+> Recursively replace sensitive entries with a special tag.
+
+```js
+import { obfuscate } from '@vates/obfuscate'
+
+obfuscate({ username: 'foo', password: 'bar' })
+// → {
+//   username: 'foo',
+//   password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
+// }
+```
+
+### `merge(newValue, oldValue)`
+
+> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
+>
+> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
+
+```js
+import { obfuscate, merge } from '@vates/obfuscate'
+
+const original = { username: 'foo', password: 'bar', token: 'baz' }
+
+// compute an obfuscated value to send to a client
+const obfuscated = obfuscate(original)
+
+// let the client update any entries
+obfuscated.token = 'qux'
+
+// merge the updated value with the original one
+merge(obfuscated, original)
+// → { username: 'foo', password: 'bar', token: 'qux' }
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/obfuscate/index.js

@@ -1,9 +1,18 @@
-import mapValues from 'lodash/mapValues.js'
+'use strict'
+
+function mapValues(object, iteratee) {
+  const result = {}
+  for (const key of Object.keys(object)) {
+    result[key] = iteratee(object[key], key, object)
+  }
+  return result
+}
 
 // this random value is used to obfuscate real data
 const OBFUSCATED_VALUE = 'obfuscated-q3oi6d9X8uenGvdLnHk2'
+exports.OBFUSCATED_VALUE = OBFUSCATED_VALUE
 
-export const merge = (newValue, oldValue) => {
+function merge(newValue, oldValue) {
   if (newValue === OBFUSCATED_VALUE) {
     return oldValue
   }
@@ -23,14 +32,16 @@ export const merge = (newValue, oldValue) => {
   const iteratee = (v, k) => merge(v, oldValue[k])
   return isArray ? newValue.map(iteratee) : mapValues(newValue, iteratee)
 }
+exports.merge = merge
 
-export const obfuscate = value => replace(value, OBFUSCATED_VALUE)
+const obfuscate = value => replace(value, OBFUSCATED_VALUE)
+exports.obfuscate = obfuscate
 
 const SENSITIVE_PARAMS = ['token', 'passphrase', /password/i, 'encryptionKey']
 const isSensitiveParam = name =>
   SENSITIVE_PARAMS.some(pattern => (typeof pattern === 'string' ? pattern === name : pattern.test(name)))
 
-export function replace(value, replacement) {
+function replace(value, replacement) {
   function helper(value, name) {
     if (typeof value === 'string' && isSensitiveParam(name)) {
       return replacement
@@ -45,3 +56,4 @@ export function replace(value, replacement) {
 
   return helper(value)
 }
+exports.replace = replace

@vates/obfuscate/package.json

@@ -0,0 +1,23 @@
+{
+  "private": false,
+  "name": "@vates/obfuscate",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/obfuscate",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/obfuscate",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/otp/index.mjs

@@ -9,6 +9,15 @@ function assert(name, value) {
   }
 }
 
+const WHITESPACES = /\s+/g
+function cleanToken(otp) {
+  // remove whitespaces
+  //
+  // some OTP clients add whitespaces for better readability (e.g. `XXX XXX`)
+  // and users might be tempted to write them
+  return otp && otp.replace(WHITESPACES, '')
+}
+
 // https://github.com/google/google-authenticator/wiki/Key-Uri-Format
 function generateUri(protocol, label, params) {
   assert('label', typeof label === 'string')
@@ -64,7 +73,7 @@ export function generateHotpUri({ counter, digits, issuer, label, secret }) {
 }
 
 export async function verifyHotp(token, opts) {
-  return token === (await generateHotp(opts))
+  return cleanToken(token) === (await generateHotp(opts))
 }
 
 function totpCounter(period = 30, timestamp = Math.floor(Date.now() / 1e3)) {
@@ -85,6 +94,7 @@ export async function verifyTotp(token, { period, timestamp, window = 1, ...opts
   const counter = totpCounter(period, timestamp)
   const end = counter + window
   opts.counter = counter - window
+  token = cleanToken(token)
   while (opts.counter <= end) {
     if (token === (await generateHotp(opts))) {
       return true

@vates/otp/index.test.mjs

@@ -1,5 +1,5 @@
 import { strict as assert } from 'node:assert'
-import { describe, it } from 'tap/mocha'
+import test from 'test'
 
 import {
   generateHotp,
@@ -11,6 +11,8 @@ import {
   verifyTotp,
 } from './index.mjs'
 
+const { describe, it } = test
+
 describe('generateSecret', function () {
   it('generates a string of 32 chars', async function () {
     const secret = generateSecret()

@vates/otp/package.json

@@ -22,7 +22,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "engines": {
     "node": ">=15"
   },
@@ -31,9 +31,9 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "tap"
+    "test": "node--test"
   },
   "devDependencies": {
-    "tap": "^16.3.0"
+    "test": "^3.3.0"
   }
 }

@vates/predicates/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { describe, it } = require('tap').mocha
+const { describe, it } = require('test')
 
 const { every, not, some } = require('./')
 

@vates/predicates/package.json

@@ -32,9 +32,9 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "tap"
+    "test": "node--test"
   },
   "devDependencies": {
-    "tap": "^16.0.1"
+    "test": "^3.3.0"
   }
 }

@vates/read-chunk/index.js

@@ -22,41 +22,41 @@ const readChunk = (stream, size) =>
   stream.errored != null
     ? Promise.reject(stream.errored)
     : stream.closed || stream.readableEnded
-    ? Promise.resolve(null)
-    : new Promise((resolve, reject) => {
-        if (size !== undefined) {
-          assert(size > 0)
+      ? Promise.resolve(null)
+      : new Promise((resolve, reject) => {
+          if (size !== undefined) {
+            assert(size > 0)
 
-          // per Node documentation:
-          // > The size argument must be less than or equal to 1 GiB.
-          assert(size < 1073741824)
-        }
+            // per Node documentation:
+            // > The size argument must be less than or equal to 1 GiB.
+            assert(size < 1073741824)
+          }
 
-        function onEnd() {
-          resolve(null)
-          removeListeners()
-        }
-        function onError(error) {
-          reject(error)
-          removeListeners()
-        }
-        function onReadable() {
-          const data = stream.read(size)
-          if (data !== null) {
-            resolve(data)
+          function onEnd() {
+            resolve(null)
             removeListeners()
           }
-        }
-        function removeListeners() {
-          stream.removeListener('end', onEnd)
-          stream.removeListener('error', onError)
-          stream.removeListener('readable', onReadable)
-        }
-        stream.on('end', onEnd)
-        stream.on('error', onError)
-        stream.on('readable', onReadable)
-        onReadable()
-      })
+          function onError(error) {
+            reject(error)
+            removeListeners()
+          }
+          function onReadable() {
+            const data = stream.read(size)
+            if (data !== null) {
+              resolve(data)
+              removeListeners()
+            }
+          }
+          function removeListeners() {
+            stream.removeListener('end', onEnd)
+            stream.removeListener('error', onError)
+            stream.removeListener('readable', onReadable)
+          }
+          stream.on('end', onEnd)
+          stream.on('error', onError)
+          stream.on('readable', onReadable)
+          onReadable()
+        })
 exports.readChunk = readChunk
 
 /**
@@ -111,42 +111,42 @@ async function skip(stream, size) {
   return stream.errored != null
     ? Promise.reject(stream.errored)
     : size === 0 || stream.closed || stream.readableEnded
-    ? Promise.resolve(0)
-    : new Promise((resolve, reject) => {
-        let left = size
-        function onEnd() {
-          resolve(size - left)
-          removeListeners()
-        }
-        function onError(error) {
-          reject(error)
-          removeListeners()
-        }
-        function onReadable() {
-          const data = stream.read()
-          left -= data === null ? 0 : data.length
-          if (left > 0) {
-            // continue to read
-          } else {
-            // if more than wanted has been read, push back the rest
-            if (left < 0) {
-              stream.unshift(data.slice(left))
-            }
-
-            resolve(size)
+      ? Promise.resolve(0)
+      : new Promise((resolve, reject) => {
+          let left = size
+          function onEnd() {
+            resolve(size - left)
             removeListeners()
           }
-        }
-        function removeListeners() {
-          stream.removeListener('end', onEnd)
-          stream.removeListener('error', onError)
-          stream.removeListener('readable', onReadable)
-        }
-        stream.on('end', onEnd)
-        stream.on('error', onError)
-        stream.on('readable', onReadable)
-        onReadable()
-      })
+          function onError(error) {
+            reject(error)
+            removeListeners()
+          }
+          function onReadable() {
+            const data = stream.read()
+            left -= data === null ? 0 : data.length
+            if (left > 0) {
+              // continue to read
+            } else {
+              // if more than wanted has been read, push back the rest
+              if (left < 0) {
+                stream.unshift(data.slice(left))
+              }
+
+              resolve(size)
+              removeListeners()
+            }
+          }
+          function removeListeners() {
+            stream.removeListener('end', onEnd)
+            stream.removeListener('error', onError)
+            stream.removeListener('readable', onReadable)
+          }
+          stream.on('end', onEnd)
+          stream.on('error', onError)
+          stream.on('readable', onReadable)
+          onReadable()
+        })
 }
 exports.skip = skip
 

@vates/task/.USAGE.md

@@ -72,6 +72,27 @@ Task.warning(message, data)
 Task.set(property, value)
 ```
 
+### `Task.run([opts], fn, ...args)`
+
+This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
+
+```js
+// options can be passed
+await Task.run({ properties: { name: 'foo' } }, fn)
+
+// arguments can be passed
+await Task.run(fn, 'foo', 'bar')
+
+// context (this) is forwarded
+await Task.run.call(object, object.method)
+
+// as a convenience, you can pass a method name directly
+await Task.run.call(object, 'method')
+
+// everything together
+await Task.run.call(object, 'method', 'foo', 'bar')
+```
+
 ### `combineEvents`
 
 Create a consolidated log from individual events.
@@ -123,7 +144,7 @@ const onProgress = makeOnProgress({
   onTaskUpdate(taskLog) {},
 })
 
-Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
+Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
 ```
 
 It can also be fed event logs directly:

@vates/task/README.md

@@ -88,6 +88,27 @@ Task.warning(message, data)
 Task.set(property, value)
 ```
 
+### `Task.run([opts], fn, ...args)`
+
+This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
+
+```js
+// options can be passed
+await Task.run({ properties: { name: 'foo' } }, fn)
+
+// arguments can be passed
+await Task.run(fn, 'foo', 'bar')
+
+// context (this) is forwarded
+await Task.run.call(object, object.method)
+
+// as a convenience, you can pass a method name directly
+await Task.run.call(object, 'method')
+
+// everything together
+await Task.run.call(object, 'method', 'foo', 'bar')
+```
+
 ### `combineEvents`
 
 Create a consolidated log from individual events.
@@ -111,7 +132,7 @@ const onProgress = makeOnProgress({
     // current status of the task as described in the previous section
     taskLog.status
 
-    // undefined or a dictionnary of properties attached to the task
+    // undefined or a dictionary of properties attached to the task
     taskLog.properties
 
     // timestamp at which the abortion was requested, undefined otherwise
@@ -139,7 +160,7 @@ const onProgress = makeOnProgress({
   onTaskUpdate(taskLog) {},
 })
 
-Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
+Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
 ```
 
 It can also be fed event logs directly:

@vates/task/index.js

@@ -3,6 +3,10 @@
 const assert = require('node:assert').strict
 const { AsyncLocalStorage } = require('node:async_hooks')
 
+function alreadyEnded() {
+  throw new Error('task has already ended')
+}
+
 // define a read-only, non-enumerable, non-configurable property
 function define(object, property, value) {
   Object.defineProperty(object, property, { value })
@@ -21,7 +25,7 @@ const asyncStorage = global[asyncStorageKey] ?? (global[asyncStorageKey] = new A
 
 const getTask = () => asyncStorage.getStore()
 
-exports.Task = class Task {
+class Task {
   static get abortSignal() {
     const task = getTask()
     if (task !== undefined) {
@@ -29,29 +33,25 @@ exports.Task = class Task {
     }
   }
 
-  static info(message, data) {
-    const task = getTask()
-    if (task !== undefined) {
-      task.#emit('info', { data, message })
+  static run(...args) {
+    let opts = args[0]
+    let fn
+    if (typeof opts === 'object') {
+      args.shift()
+      fn = args[0]
+    } else {
+      fn = opts
+      opts = undefined
     }
-  }
+    args.shift()
 
-  static run(opts, fn) {
-    return new this(opts).run(fn)
-  }
+    const thisArg = this !== Task ? this : undefined
 
-  static set(name, value) {
-    const task = getTask()
-    if (task !== undefined) {
-      task.#emit('property', { name, value })
+    if (typeof fn !== 'function') {
+      fn = this[fn]
     }
-  }
 
-  static warning(message, data) {
-    const task = getTask()
-    if (task !== undefined) {
-      task.#emit('warning', { data, message })
-    }
+    return new Task(opts).run(() => fn.apply(thisArg, args))
   }
 
   static wrap(opts, fn) {
@@ -110,9 +110,7 @@ exports.Task = class Task {
         this.#emit('abortionRequested', { reason: signal.reason })
 
         if (!this.#running) {
-          const status = FAILURE
-          this.#status = status
-          this.#emit('end', { result: signal.reason, status })
+          this.#end(FAILURE, signal.reason)
         }
       }
     })
@@ -123,12 +121,26 @@ exports.Task = class Task {
   }
 
   #emit(type, data) {
+    assert.equal(this.#startData, undefined, 'task has not started yet')
+
     data.id = this.id
     data.timestamp = Date.now()
     data.type = type
     this.#onProgress(data)
   }
 
+  #end(status, result) {
+    assert.equal(this.#status, PENDING)
+
+    this.#status = status
+    this.#emit('end', { status, result })
+    this.#onProgress = alreadyEnded
+  }
+
+  info(message, data) {
+    this.#emit('info', { data, message })
+  }
+
   #maybeStart() {
     const startData = this.#startData
     if (startData !== undefined) {
@@ -140,8 +152,7 @@ exports.Task = class Task {
   async run(fn) {
     const result = await this.runInside(fn)
     if (this.status === PENDING) {
-      this.#status = SUCCESS
-      this.#emit('end', { status: SUCCESS, result })
+      this.#end(SUCCESS, result)
     }
     return result
   }
@@ -155,18 +166,27 @@ exports.Task = class Task {
     this.#maybeStart()
 
     try {
-      const result = await asyncStorage.run(this, fn)
+      const result = await asyncStorage.run(this, fn, this)
       this.#running = false
       return result
     } catch (result) {
-      const status = FAILURE
-
-      this.#status = status
-      this.#emit('end', { status, result })
+      this.#end(FAILURE, result)
       throw result
     }
   }
 
+  set(name, value) {
+    assert.equal(this.status, PENDING)
+
+    this.#emit('property', { name, value })
+  }
+
+  warning(message, data) {
+    assert.equal(this.status, PENDING)
+
+    this.#emit('warning', { data, message })
+  }
+
   wrap(fn) {
     const task = this
     return function taskRun() {
@@ -181,3 +201,15 @@ exports.Task = class Task {
     }
   }
 }
+exports.Task = Task
+
+// setup static aliases
+for (const name of ['info', 'set', 'warning']) {
+  const method = Task.prototype[name]
+  Task[name] = function () {
+    const task = getTask()
+    if (task !== undefined) {
+      return method.apply(task, arguments)
+    }
+  }
+}

@vates/task/index.test.js

@@ -3,6 +3,7 @@
 const assert = require('node:assert').strict
 const { describe, it } = require('test')
 
+const { makeOnProgress } = require('./combineEvents.js')
 const { Task } = require('./index.js')
 
 const noop = Function.prototype
@@ -162,6 +163,44 @@ describe('Task', function () {
     })
   })
 
+  describe('.run()', function () {
+    function fn(...args) {
+      Task.set('baz', 'qux')
+
+      return [this, ...args]
+    }
+    const obj = { fn }
+
+    it('accepts optional options', async function () {
+      assert.deepEqual(
+        await Task.run(
+          {
+            properties: { foo: 'bar' },
+            onProgress: makeOnProgress({
+              onRootTaskEnd(log) {
+                assert.deepEqual(log.properties, { __proto__: null, foo: 'bar', baz: 'qux' })
+              },
+            }),
+          },
+          fn
+        ),
+        [undefined]
+      )
+    })
+
+    it('accepts optional arguments', async function () {
+      assert.deepEqual(await Task.run(fn, 'foo', 'bar'), [undefined, 'foo', 'bar'])
+    })
+
+    it('accepts optional context', async function () {
+      assert.deepEqual(await Task.run.call('foo', fn), ['foo'])
+    })
+
+    it('accepts a method name instead of a function', async function () {
+      assert.deepEqual(await Task.run.call(obj, 'fn'), [obj])
+    })
+  })
+
   describe('.set()', function () {
     it('does nothing when run outside a task', function () {
       Task.set('progress', 10)
@@ -180,6 +219,31 @@ describe('Task', function () {
     })
   })
 
+  describe('#set()', function () {
+    const name = 'progress'
+    const value = 10
+
+    it('throws when the task is not started', function () {
+      const task = createTask()
+      assert.throws(() => task.set(name, value), { message: 'task has not started yet' })
+    })
+
+    it(`emits an property message`, async function () {
+      const task = createTask()
+      await task.run(async () => {
+        await Task.run(() => {
+          task.set(name, value)
+
+          assertEvent(task, {
+            name,
+            type: 'property',
+            value,
+          })
+        })
+      })
+    })
+  })
+
   describe('.warning()', function () {
     it('does nothing when run outside a task', function () {
       Task.warning('foo')
@@ -198,6 +262,30 @@ describe('Task', function () {
     })
   })
 
+  for (const type of ['info', 'warning']) {
+    describe(`#${type}()`, function () {
+      it('throws when the task is not started', function () {
+        const task = createTask()
+        assert.throws(() => task[type]('foo'), { message: 'task has not started yet' })
+      })
+
+      it(`emits an ${type} message`, async function () {
+        const task = createTask()
+        await task.run(async () => {
+          await Task.run(() => {
+            task[type]('foo')
+
+            assertEvent(task, {
+              data: undefined,
+              message: 'foo',
+              type,
+            })
+          })
+        })
+      })
+    })
+  }
+
   describe('#id', function () {
     it('can be set', function () {
       const task = createTask()

@vates/task/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "engines": {
     "node": ">=14"
   },

@vates/xml-rpc/.USAGE.md

@@ -0,0 +1,128 @@
+### Formatting
+
+```js
+import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
+
+// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
+xmlRpcFormatter.format_methodCall({
+  methodName: 'examples.getStateName',
+  params: [40],
+})
+// → {
+//   name: 'methodCall',
+//   children: [
+//     {
+//       name: 'methodName',
+//       children: [ 'examples.getStateName' ]
+//     },
+//     {
+//       name: 'params',
+//       children: [
+//         {
+//           name: 'param',
+//           children: [ { name: 'value', children: [ '40' ] } ]
+//         }
+//       ]
+//     }
+//   ]
+// }
+
+// There is a dedicated method to format a JS value
+xmlRpcFormatter.format_js(new Date(900684535000))
+// → {
+//   name: 'dateTime.iso8601',
+//   children: [ '1998-07-17T14:08:55.000Z' ]
+// }
+```
+
+Formatting XML is out of this library's scope, but you can achieve it easily with
+`@vates/xml`:
+
+```js
+import { formatXml } from '@vates/xml/format'
+
+// it can now be passed to the XML-RPC parser
+const tree = xmlRpcFormatter.format_methodCall({
+  methodName: 'examples.getStateName',
+  params: [40],
+})
+
+// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
+const xml = formatXml(tree, { indent: 0 })
+```
+
+### Parsing
+
+```js
+import { xmlRpcParser } from '@vates/xml-rpc/parser'
+
+xmlRpcParser.parse({
+  name: 'methodResponse',
+  children: [
+    {
+      name: 'params',
+      children: [
+        {
+          name: 'param',
+          children: [
+            {
+              name: 'value',
+              children: [
+                {
+                  name: 'string',
+                  children: ['South Dakota'],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+  ],
+})
+// { params: [ 'South Dakota' ] }
+
+// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
+xmlRpcParser.parse_array({
+  name: 'array',
+  children: [
+    {
+      name: 'data',
+      children: [
+        {
+          name: 'value',
+          children: [{ name: 'boolean', children: ['1'] }],
+        },
+        {
+          name: 'value',
+          children: [{ name: 'int', children: ['42'] }],
+        },
+      ],
+    },
+  ],
+})
+// [ true, 42 ]
+```
+
+Parsing XML is out of this library's scope, but you can achieve it easily with
+`@vates/xml`:
+
+```js
+import { parseXml } from '@vates/xml/parse'
+
+const xml = `
+<?xml version="1.0"?>
+<methodResponse>
+  <params>
+    <param>
+        <value><string>South Dakota</string></value>
+    </param>
+  </params>
+</methodResponse>
+`
+
+const tree = parseXml(xml)
+
+// it can now be passed to the XML-RPC parser
+xmlRpcParser.parse(tree)
+```

@vates/xml-rpc/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/xml-rpc/README.md

@@ -0,0 +1,161 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/xml-rpc
+
+[![Package Version](https://badgen.net/npm/v/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc) ![License](https://badgen.net/npm/license/@vates/xml-rpc) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml-rpc)](https://bundlephobia.com/result?p=@vates/xml-rpc) [![Node compatibility](https://badgen.net/npm/node/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc)
+
+> Parse and format XML-RPC messages, do not handle transport or XML
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/xml-rpc):
+
+```sh
+npm install --save @vates/xml-rpc
+```
+
+## Usage
+
+### Formatting
+
+```js
+import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
+
+// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
+xmlRpcFormatter.format_methodCall({
+  methodName: 'examples.getStateName',
+  params: [40],
+})
+// → {
+//   name: 'methodCall',
+//   children: [
+//     {
+//       name: 'methodName',
+//       children: [ 'examples.getStateName' ]
+//     },
+//     {
+//       name: 'params',
+//       children: [
+//         {
+//           name: 'param',
+//           children: [ { name: 'value', children: [ '40' ] } ]
+//         }
+//       ]
+//     }
+//   ]
+// }
+
+// There is a dedicated method to format a JS value
+xmlRpcFormatter.format_js(new Date(900684535000))
+// → {
+//   name: 'dateTime.iso8601',
+//   children: [ '1998-07-17T14:08:55.000Z' ]
+// }
+```
+
+Formatting XML is out of this library's scope, but you can achieve it easily with
+`@vates/xml`:
+
+```js
+import { formatXml } from '@vates/xml/format'
+
+// it can now be passed to the XML-RPC parser
+const tree = xmlRpcFormatter.format_methodCall({
+  methodName: 'examples.getStateName',
+  params: [40],
+})
+
+// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
+const xml = formatXml(tree, { indent: 0 })
+```
+
+### Parsing
+
+```js
+import { xmlRpcParser } from '@vates/xml-rpc/parser'
+
+xmlRpcParser.parse({
+  name: 'methodResponse',
+  children: [
+    {
+      name: 'params',
+      children: [
+        {
+          name: 'param',
+          children: [
+            {
+              name: 'value',
+              children: [
+                {
+                  name: 'string',
+                  children: ['South Dakota'],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+  ],
+})
+// { params: [ 'South Dakota' ] }
+
+// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
+xmlRpcParser.parse_array({
+  name: 'array',
+  children: [
+    {
+      name: 'data',
+      children: [
+        {
+          name: 'value',
+          children: [{ name: 'boolean', children: ['1'] }],
+        },
+        {
+          name: 'value',
+          children: [{ name: 'int', children: ['42'] }],
+        },
+      ],
+    },
+  ],
+})
+// [ true, 42 ]
+```
+
+Parsing XML is out of this library's scope, but you can achieve it easily with
+`@vates/xml`:
+
+```js
+import { parseXml } from '@vates/xml/parse'
+
+const xml = `
+<?xml version="1.0"?>
+<methodResponse>
+  <params>
+    <param>
+        <value><string>South Dakota</string></value>
+    </param>
+  </params>
+</methodResponse>
+`
+
+const tree = parseXml(xml)
+
+// it can now be passed to the XML-RPC parser
+xmlRpcParser.parse(tree)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/xml-rpc/formatter.js

@@ -0,0 +1,104 @@
+'use strict'
+
+function node(name, attributes, children) {
+  if (Array.isArray(attributes)) {
+    children = attributes
+    attributes = undefined
+  }
+
+  return { name, attributes, children }
+}
+
+class XmlRpcFormatter {
+  format_js(value) {
+    const type = typeof value
+    if (type === 'object') {
+      return value === null
+        ? this.format_nil()
+        : value instanceof Date
+          ? this['format_dateTime.iso8601'](value)
+          : Array.isArray(value)
+            ? this.format_array(value)
+            : Buffer.isBuffer(value)
+              ? this.format_base64(value)
+              : this.format_struct(value)
+    }
+
+    return this['format_' + type](value)
+  }
+
+  format_array(values) {
+    return node('array', [node('data', values.map(this.format_value, this))])
+  }
+
+  format_base64(buf) {
+    return node('base64', [buf.toString('base64')])
+  }
+
+  format_boolean(bool) {
+    return node('boolean', [bool ? '1' : '0'])
+  }
+
+  'format_dateTime.iso8601'(date) {
+    return node('dateTime.iso8601', [date.toISOString()])
+  }
+
+  format_double(str) {
+    return node('double', [String(str)])
+  }
+
+  format_fault(val) {
+    return node('fault', [this.format_value(val)])
+  }
+
+  format_i4(str) {
+    return node('i4', [String(str)])
+  }
+
+  format_int(str) {
+    return node('int', [String(str)])
+  }
+
+  format_member([name, value]) {
+    return node('member', [node('name', [name]), this.format_value(value)])
+  }
+
+  format_methodCall({ methodName, params }) {
+    return node('methodCall', [node('methodName', [methodName]), this.format_params(params)])
+  }
+
+  format_methodResponse({ fault, params }) {
+    return node('methodResponse', [fault !== undefined ? this.format_fault(fault) : this.format_params(params)])
+  }
+
+  format_nil() {
+    return node('nil')
+  }
+
+  format_number(num) {
+    return Number.isInteger(num) ? this.format_int(num) : this.format_double(num)
+  }
+
+  format_param(value) {
+    return node('param', [this.format_value(value)])
+  }
+
+  format_params(params) {
+    return node('params', params.map(this.format_param, this))
+  }
+
+  format_string(str) {
+    return str
+  }
+
+  format_struct(struct) {
+    return node('struct', Object.entries(struct).map(this.format_member, this))
+  }
+
+  format_value(value) {
+    return node('value', [this.format_js(value)])
+  }
+}
+exports.Formatter = XmlRpcFormatter
+
+exports.xmlRpcFormatter = new XmlRpcFormatter()

@vates/xml-rpc/package.json

@@ -0,0 +1,33 @@
+{
+  "private": false,
+  "name": "@vates/xml-rpc",
+  "description": "Parse and format XML-RPC messages, do not handle transport or XML",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml-rpc",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/xml-rpc",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=12"
+  },
+  "devDependencies": {
+    "@vates/xml": "^2.0.0",
+    "test": "^3.3.0"
+  },
+  "exports": {
+    "./formatter": "./formatter.js",
+    "./parser": "./parser.js"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "node--test"
+  }
+}

@vates/xml-rpc/parser.js

@@ -0,0 +1,133 @@
+'use strict'
+
+function toValue(node) {
+  const obj = { __proto__: null }
+  for (const child of node.children) {
+    if (typeof child === 'string') {
+      return child
+    }
+
+    const { name } = child
+    if (Object.hasOwn(obj, name)) {
+      const prev = obj[name]
+      if (Array.isArray(obj, name)) {
+        prev.push(child)
+      } else {
+        obj[name] = [prev, child]
+      }
+    } else {
+      obj[name] = child
+    }
+  }
+  return obj
+}
+
+// https://en.wikipedia.org/wiki/XML-RPC#Data_types
+class XmlRpcParser {
+  parse(node) {
+    return this['parse_' + node.name](node)
+  }
+
+  parse_array({ children: [data] }) {
+    return data.children.map(this.parse_value, this)
+  }
+
+  parse_base64({ children: [str] }) {
+    return Buffer.from(str, 'base64')
+  }
+
+  parse_boolean({ children: [str] }) {
+    return Boolean(Number(str))
+  }
+
+  'parse_dateTime.iso8601'({ children: [str] }) {
+    const date = new Date(str)
+
+    // date successfully parsed
+    if (!Number.isNaN(+date)) {
+      return date
+    }
+
+    // XML-RPC dates may use YYYYMMDD instead of YYYY-MM-DD which ECMAScript
+    // does not parse
+    return new Date(str.slice(0, 4) + '-' + str.slice(4, 6) + '-' + str.slice(6))
+  }
+
+  parse_double({ children: [str] }) {
+    return Number(str)
+  }
+
+  parse_fault({ children: [value] }) {
+    return this.parse_value(value)
+  }
+
+  parse_i4({ children: [str] }) {
+    return Number(str)
+  }
+
+  parse_int({ children: [str] }) {
+    return Number(str)
+  }
+
+  parse_member(member) {
+    member = toValue(member)
+    return [member.name.children[0], this.parse_value(member.value)]
+  }
+
+  parse_methodCall(methodCall) {
+    methodCall = toValue(methodCall)
+    return {
+      methodName: methodCall.methodName.children[0],
+      params: this.parse_params(methodCall.params),
+    }
+  }
+
+  parse_methodResponse(methodResponse) {
+    methodResponse = toValue(methodResponse)
+
+    const { fault } = methodResponse
+    if (fault !== undefined) {
+      return { fault: this.parse_fault(fault) }
+    }
+
+    return { params: this.parse_params(methodResponse.params) }
+  }
+
+  parse_nil() {
+    return null
+  }
+
+  parse_param(param) {
+    return this.parse_value(param.children[0])
+  }
+
+  parse_params(params) {
+    return params.children.map(this.parse_param, this)
+  }
+
+  parse_string({ children: [str] }) {
+    return str
+  }
+
+  parse_struct(struct) {
+    return Object.fromEntries(struct.children.map(this.parse_member, this))
+  }
+
+  parse_value({ children }) {
+    if (children.length === 0) {
+      return ''
+    }
+
+    const child = children[0]
+
+    // can be a plain string
+    if (typeof child === 'string') {
+      return child
+    }
+
+    return this.parse(child)
+  }
+}
+exports.XmlRpcParser = XmlRpcParser
+
+exports.xmlRpcParser = new XmlRpcParser()

@vates/xml-rpc/test/formatter.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const { formatXml } = require('@vates/xml/format')
+const { join } = require('node:path')
+const { parseXml } = require('@vates/xml/parse')
+const { readFileSync } = require('node:fs')
+const { xmlRpcFormatter } = require('@vates/xml-rpc/formatter')
+const assert = require('node:assert/strict')
+const test = require('test')
+
+const methodCall = require('./methodCall.js')
+
+test(function () {
+  const xmlTree = parseXml(readFileSync(join(__dirname, 'methodCall.xml')))
+
+  const xmlRpc = xmlRpcFormatter.format_methodCall(methodCall)
+
+  assert.deepEqual(parseXml(formatXml(xmlRpc)), xmlTree)
+})

@vates/xml-rpc/test/methodCall.js

@@ -0,0 +1,16 @@
+'use strict'
+
+module.exports = {
+  methodName: 'examples.getStateName',
+  params: [
+    [1404, 'Something here'],
+    Buffer.from([121, 111, 117, 32, 99, 97, 110, 39, 116, 32, 114, 101, 97, 100, 32, 116, 104, 105, 115, 33]),
+    true,
+    new Date('1998-07-17T14:08:55.000Z'),
+    -12.53,
+    42,
+    'Hello world!',
+    { foo: 1, bar: 2 },
+    null,
+  ],
+}

@vates/xml-rpc/test/methodCall.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<methodCall>
+  <methodName>examples.getStateName</methodName>
+  <params>
+    <param>
+      <value>
+        <array>
+          <data>
+            <value>
+              <int>1404</int>
+            </value>
+            <value>
+              Something here
+            </value>
+          </data>
+        </array>
+      </value>
+    </param>
+    <param>
+      <value>
+        <base64>eW91IGNhbid0IHJlYWQgdGhpcyE=</base64>
+      </value>
+    </param>
+    <param>
+      <value>
+        <boolean>1</boolean>
+      </value>
+    </param>
+    <param>
+      <value>
+        <dateTime.iso8601>1998-07-17T14:08:55.000Z</dateTime.iso8601>
+      </value>
+    </param>
+    <param>
+      <value>
+        <double>-12.53</double>
+      </value>
+    </param>
+    <param>
+      <value>
+        <int>42</int>
+      </value>
+    </param>
+    <param>
+      <value>
+        Hello world!
+      </value>
+    </param>
+    <param>
+      <value>
+        <struct>
+          <member>
+            <name>foo</name>
+            <value>
+              <int>1</int>
+            </value>
+          </member>
+          <member>
+            <name>bar</name>
+            <value>
+              <int>2</int>
+            </value>
+          </member>
+        </struct>
+      </value>
+    </param>
+    <param>
+      <value>
+        <nil />
+      </value>
+    </param>
+  </params>
+</methodCall>

@vates/xml-rpc/test/parser.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const { join } = require('node:path')
+const { parseXml } = require('@vates/xml/parse')
+const { readFileSync } = require('node:fs')
+const { xmlRpcParser } = require('@vates/xml-rpc/parser')
+const assert = require('node:assert/strict')
+const test = require('test')
+
+const methodCall = require('./methodCall.js')
+
+test('parser.parse()', function () {
+  const xml = String(readFileSync(join(__dirname, 'methodCall.xml')))
+  const xmlRpc = parseXml(xml)
+
+  assert.deepEqual(xmlRpcParser.parse(xmlRpc), methodCall)
+})
+
+test('datetime without dashes nor milliseconds', function () {
+  assert.deepEqual(
+    xmlRpcParser.parse({ name: 'dateTime.iso8601', children: ['19980717T14:08:55Z'] }),
+    new Date('1998-07-17T14:08:55Z')
+  )
+})
+
+test('i4 support', function () {
+  assert.equal(xmlRpcParser.parse({ name: 'i4', children: ['42'] }), 42)
+})
+
+test('string support', function () {
+  assert.equal(xmlRpcParser.parse({ name: 'string', children: ['foo'] }), 'foo')
+})

@vates/xml/.USAGE.md

@@ -0,0 +1,56 @@
+### `parseXml(xml, [opts])`
+
+> Based on [`sax`](https://www.npmjs.com/package/sax)
+
+`opts` is an optional object which can contain the following options:
+
+- `normalize = true`: if true, then turn any whitespace into a single space
+- `strict = true`: whether or not to be a jerk
+- `trim = true`: whether or not to trim text nodes
+
+```js
+import { parseXml } from '@xen-orchestra/xml/parse'
+
+const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
+<tag1 attr1="value1" attr2="value2">
+  Text &amp; entities
+  <tag2 />
+  <tag2 />
+  <ns:tag3 />
+</tag1>
+`)
+// → {
+//   name: 'tag1',
+//   attributes: { attr1: 'value1', attr2: 'value2' },
+//   children: [
+//     'Text & entities',
+//     { name: 'tag2', attributes: {}, children: [] },
+//     { name: 'tag2', attributes: {}, children: [] },
+//     { name: 'ns:tag3', attributes: {}, children: [] }
+//   ]
+// }
+```
+
+### `formatXml(tree, [opts])`
+
+`opts` is an optional object which can contain the following options:
+
+- `includeDeclaration = true`: whether to include an XML declaration
+- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
+
+```js
+import { formatXml } from '@xen-orchestra/xml/format'
+
+formatXml({
+  name: 'tag1',
+  attributes: { attr1: 'value1', attr2: 'value2' },
+  children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
+})
+// → <?xml version="1.0" encoding="UTF-8"?>
+// <tag1 attr1="value1" attr2="value2">
+//   Text &amp; entities
+//   <tag2 />
+//   <tag2 />
+//   <ns:tag3 />
+// </tag1>
+```

@vates/xml/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/xml/README.md

@@ -0,0 +1,89 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/xml
+
+[![Package Version](https://badgen.net/npm/v/@vates/xml)](https://npmjs.org/package/@vates/xml) ![License](https://badgen.net/npm/license/@vates/xml) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml)](https://bundlephobia.com/result?p=@vates/xml) [![Node compatibility](https://badgen.net/npm/node/@vates/xml)](https://npmjs.org/package/@vates/xml)
+
+> Simple XML formatting/parsing
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/xml):
+
+```sh
+npm install --save @vates/xml
+```
+
+## Usage
+
+### `parseXml(xml, [opts])`
+
+> Based on [`sax`](https://www.npmjs.com/package/sax)
+
+`opts` is an optional object which can contain the following options:
+
+- `normalize = true`: if true, then turn any whitespace into a single space
+- `strict = true`: whether or not to be a jerk
+- `trim = true`: whether or not to trim text nodes
+
+```js
+import { parseXml } from '@xen-orchestra/xml/parse'
+
+const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
+<tag1 attr1="value1" attr2="value2">
+  Text &amp; entities
+  <tag2 />
+  <tag2 />
+  <ns:tag3 />
+</tag1>
+`)
+// → {
+//   name: 'tag1',
+//   attributes: { attr1: 'value1', attr2: 'value2' },
+//   children: [
+//     'Text & entities',
+//     { name: 'tag2', attributes: {}, children: [] },
+//     { name: 'tag2', attributes: {}, children: [] },
+//     { name: 'ns:tag3', attributes: {}, children: [] }
+//   ]
+// }
+```
+
+### `formatXml(tree, [opts])`
+
+`opts` is an optional object which can contain the following options:
+
+- `includeDeclaration = true`: whether to include an XML declaration
+- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
+
+```js
+import { formatXml } from '@xen-orchestra/xml/format'
+
+formatXml({
+  name: 'tag1',
+  attributes: { attr1: 'value1', attr2: 'value2' },
+  children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
+})
+// → <?xml version="1.0" encoding="UTF-8"?>
+// <tag1 attr1="value1" attr2="value2">
+//   Text &amp; entities
+//   <tag2 />
+//   <tag2 />
+//   <ns:tag3 />
+// </tag1>
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/xml/cli.js

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+'use strict'
+
+const { inspect } = require('node:util')
+const { formatXml } = require('@vates/xml/format')
+const { parseXml } = require('@vates/xml/parse')
+const { readFileSync } = require('node:fs')
+
+function log(val) {
+  process.stdout.write(inspect(val, false, null, true))
+  process.stdout.write('\n')
+}
+
+function main([inputPath = 0]) {
+  const input = readFileSync(inputPath)
+
+  // attempt to parse from JSON
+  let data
+  try {
+    data = JSON.parse(input)
+  } catch (error) {
+    // fallback to XML
+    log(parseXml(input))
+    return
+  }
+
+  process.stdout.write(formatXml(data))
+}
+main(process.argv.slice(2))

@vates/xml/format.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const ENTITIES = {
+  '"': '"',
+  '&': '&',
+  '<': '&lt;',
+  '>': '&gt;',
+  "'": '&apos;',
+}
+const RE = new RegExp(Object.keys(ENTITIES).join('|'), 'g')
+function replace(match) {
+  return ENTITIES[match]
+}
+function escape(str) {
+  return str.replace(RE, replace)
+}
+
+function formatNode(node, depth) {
+  const indent = this.indent.repeat(depth)
+
+  if (typeof node === 'object') {
+    const line = [indent, '<', node.name]
+    const { attributes } = node
+    if (attributes !== undefined) {
+      for (const name of Object.keys(attributes)) {
+        line.push(' ', name, '="', escape(attributes[name]), '"')
+      }
+    }
+
+    const { children } = node
+    if (children === undefined || children.length === 0) {
+      line.push(' />')
+      this.lines.push(line.join(''))
+    } else {
+      line.push('>')
+      this.lines.push(line.join(''))
+
+      for (const child of children) {
+        formatNode.call(this, child, depth + 1)
+      }
+
+      this.lines.push(indent + '</' + node.name + '>')
+    }
+  } else {
+    // string (or scalar)
+    this.lines.push(indent + escape(String(node)))
+  }
+}
+
+exports.formatXml = function formatXml(tree, { includeDeclaration = true, indent = 2 } = {}) {
+  const lines = []
+
+  if (includeDeclaration) {
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>')
+  }
+
+  const ctx = {
+    lines,
+    indent: typeof indent === 'number' ? ' '.repeat(indent) : indent,
+  }
+  formatNode.call(ctx, tree, 0)
+
+  return lines.join(indent !== 0 && indent !== '' ? '\n' : '')
+}

@vates/xml/package.json

@@ -0,0 +1,35 @@
+{
+  "private": false,
+  "name": "@vates/xml",
+  "description": "Simple XML formatting/parsing",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/xml",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "2.0.0",
+  "engines": {
+    "node": ">=14"
+  },
+  "dependencies": {
+    "sax": "^1.3.0"
+  },
+  "devDependencies": {
+    "test": "^3.3.0"
+  },
+  "exports": {
+    "./format": "./format.js",
+    "./parse": "./parse.js"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "node--test"
+  }
+}

@vates/xml/parse.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const sax = require('sax')
+
+exports.parseXml = function parseXml(xml, { normalize = true, strict = true, trim = true } = {}) {
+  const stack = [{ children: [] }]
+
+  const parser = sax.parser(strict, { normalize, trim })
+  parser.ontext = text => {
+    stack[stack.length - 1].children.push(text)
+  }
+  parser.onopentag = ({ name, attributes }) => {
+    stack.push({ name, attributes, children: [] })
+  }
+  parser.onclosetag = () => {
+    const node = stack.pop()
+    stack[stack.length - 1].children.push(node)
+  }
+
+  parser.write(xml).close()
+
+  return stack[0].children[0]
+}

@vates/xml/test/data.json

@@ -0,0 +1,25 @@
+{
+  "name": "tag1",
+  "attributes": {
+    "attr1": "value1",
+    "attr2": "value2"
+  },
+  "children": [
+    "Text & entities",
+    {
+      "name": "tag2",
+      "attributes": {},
+      "children": []
+    },
+    {
+      "name": "tag2",
+      "attributes": {},
+      "children": []
+    },
+    {
+      "name": "ns:tag3",
+      "attributes": {},
+      "children": []
+    }
+  ]
+}

@vates/xml/test/data.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<tag1 attr1="value1" attr2="value2">
+  Text &amp; entities
+  <tag2 />
+  <tag2 />
+  <ns:tag3 />
+</tag1>

@vates/xml/test/format.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const { formatXml } = require('@vates/xml/format')
+const { join } = require('node:path')
+const { readFileSync } = require('node:fs')
+const assert = require('node:assert/strict')
+const test = require('test')
+
+test('format()', function () {
+  const xml = String(readFileSync(join(__dirname, 'data.xml')))
+  const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
+
+  assert.equal(formatXml(tree) + '\n', xml)
+})
+
+test('supports missing attributes and children', function () {
+  assert.equal(formatXml({ name: 'foo' }, { includeDeclaration: false }), '<foo />')
+})

@vates/xml/test/parse.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const { parseXml } = require('@vates/xml/parse')
+const { readFileSync } = require('node:fs')
+const { join } = require('node:path')
+const assert = require('node:assert/strict')
+const test = require('test')
+
+test('parse()', function () {
+  const xml = String(readFileSync(join(__dirname, 'data.xml')))
+  const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
+
+  assert.deepEqual(parseXml(xml), tree)
+})

@xen-orchestra/async-map/package.json

@@ -35,7 +35,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^16.0.0",
+    "sinon": "^18.0.0",
     "test": "^3.2.1"
   }
 }

@xen-orchestra/audit-core/index.js

@@ -130,6 +130,10 @@ class AuditCore {
     return nValid
   }
 
+  get(id) {
+    return this._storage.get(id)
+  }
+
   async *getFrom(newest) {
     const storage = this._storage
 

@xen-orchestra/audit-core/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { afterEach, describe, it } = require('tap').mocha
+const { afterEach, describe, it } = require('test')
 
 const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')
 

@xen-orchestra/audit-core/package.json

@@ -7,16 +7,16 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.2.3",
+  "version": "0.3.0",
   "engines": {
     "node": ">=14"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "tap --lines 67 --functions 92 --branches 52 --statements 67"
+    "test": "node--test"
   },
   "dependencies": {
-    "@vates/decorate-with": "^2.0.0",
+    "@vates/decorate-with": "^2.1.0",
     "@xen-orchestra/log": "^0.6.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"
@@ -28,6 +28,6 @@
     "url": "https://vates.fr"
   },
   "devDependencies": {
-    "tap": "^16.0.1"
+    "test": "^3.3.0"
   }
 }

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.43.0",
-    "@xen-orchestra/fs": "^4.1.0",
+    "@xen-orchestra/backups": "^0.52.2",
+    "@xen-orchestra/fs": "^4.1.7",
     "filenamify": "^6.0.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.0.13",
+  "version": "1.0.22",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -4,23 +4,229 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { importIncrementalVm } from './_incrementalVm.mjs'
 import { Task } from './Task.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
+import { VhdNegative, VhdSynthetic } from 'vhd-lib'
+import { decorateClass } from '@vates/decorate-with'
+import { createLogger } from '@xen-orchestra/log'
+import { dirname, join } from 'node:path'
+import pickBy from 'lodash/pickBy.js'
+import { defer } from 'golike-defer'
 
+const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
+async function resolveUuid(xapi, cache, uuid, type) {
+  if (uuid == null) {
+    return uuid
+  }
+  const ref = cache.get(uuid)
+  if (ref === undefined) {
+    cache.set(uuid, xapi.call(`${type}.get_by_uuid`, uuid))
+  }
+  return cache.get(uuid)
+}
 export class ImportVmBackup {
-  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs = {} } = {} }) {
+  constructor({
+    adapter,
+    metadata,
+    srUuid,
+    xapi,
+    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
+  }) {
     this._adapter = adapter
-    this._importIncrementalVmSettings = { newMacAddresses, mapVdisSrs }
+    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
   }
 
+  async #getPathOfVdiSnapshot(snapshotUuid) {
+    const metadata = this._metadata
+    if (this._pathToVdis === undefined) {
+      const backups = await this._adapter.listVmBackups(
+        this._metadata.vm.uuid,
+        ({ mode, timestamp }) => mode === 'delta' && timestamp >= metadata.timestamp
+      )
+      const map = new Map()
+      for (const backup of backups) {
+        for (const [vdiRef, vdi] of Object.entries(backup.vdis)) {
+          map.set(vdi.uuid, backup.vhds[vdiRef])
+        }
+      }
+      this._pathToVdis = map
+    }
+    return this._pathToVdis.get(snapshotUuid)
+  }
+
+  async _reuseNearestSnapshot($defer, ignoredVdis) {
+    const metadata = this._metadata
+    const { mapVdisSrs } = this._importIncrementalVmSettings
+    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const streams = {}
+    const metdataDir = dirname(metadata._filename)
+    const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
+
+    for (const [vdiRef, vdi] of Object.entries(vdis)) {
+      const vhdPath = join(metdataDir, vhds[vdiRef])
+
+      let xapiDisk
+      try {
+        xapiDisk = await this._xapi.getRecordByUuid('VDI', vdi.$snapshot_of$uuid)
+      } catch (err) {
+        // if this disk is not present anymore, fall back to default restore
+        warn(err)
+      }
+
+      let snapshotCandidate, backupCandidate
+      if (xapiDisk !== undefined) {
+        debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
+        for (const snapshotRef of xapiDisk.snapshots) {
+          const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
+          debug('handling snapshot', { snapshot })
+
+          // take only the first snapshot
+          if (snapshotCandidate && snapshotCandidate.snapshot_time < snapshot.snapshot_time) {
+            debug('already got a better candidate')
+            continue
+          }
+
+          // have a corresponding backup more recent than metadata ?
+          const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
+          if (pathToSnapshotData === undefined) {
+            debug('no backup linked to this snaphot')
+            continue
+          }
+          if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
+            debug('not restored on the same SR', { snapshotSr: snapshot.$SR.uuid, mapVdisSrs, srUuid: this._srUuid })
+            continue
+          }
+
+          debug('got a candidate', pathToSnapshotData)
+
+          snapshotCandidate = snapshot
+          backupCandidate = pathToSnapshotData
+        }
+      }
+
+      let stream
+      const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
+      if (vhdPath === backupWithSnapshotPath) {
+        // all the data are already on the host
+        debug('direct reuse of a snapshot')
+        stream = null
+        vdis[vdiRef].baseVdi = snapshotCandidate
+        // go next disk , we won't use this stream
+        continue
+      }
+
+      let disposableDescendants
+
+      const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
+
+      // this will also clean if another disk of this VM backup fails
+      // if user really only need to restore non failing disks he can retry with ignoredVdis
+      let disposed = false
+      const disposeOnce = async () => {
+        if (!disposed) {
+          disposed = true
+          try {
+            await disposableDescendants?.dispose()
+            await disposableSynthetic?.dispose()
+          } catch (error) {
+            warn('openVhd: failed to dispose VHDs', { error })
+          }
+        }
+      }
+      $defer.onFailure(() => disposeOnce())
+
+      const parentVhd = disposableSynthetic.value
+      await parentVhd.readBlockAllocationTable()
+      debug('got vhd synthetic of parents', parentVhd.length)
+
+      if (snapshotCandidate !== undefined) {
+        try {
+          debug('will try to use differential restore', {
+            backupWithSnapshotPath,
+            vhdPath,
+            vdiRef,
+          })
+
+          disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
+            until: vhdPath,
+          })
+          const descendantsVhd = disposableDescendants.value
+          await descendantsVhd.readBlockAllocationTable()
+          debug('got vhd synthetic of descendants')
+          const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
+          debug('got vhd negative')
+
+          // update the stream with the negative vhd stream
+          stream = await negativeVhd.stream()
+          vdis[vdiRef].baseVdi = snapshotCandidate
+        } catch (error) {
+          // can be a broken VHD chain, a vhd chain with a key backup, ....
+          // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
+          warn(`can't use differential restore`, { error })
+          disposableDescendants?.dispose()
+        }
+      }
+      // didn't make a negative stream : fallback to classic stream
+      if (stream === undefined) {
+        debug('use legacy restore')
+        stream = await parentVhd.stream()
+      }
+
+      stream.on('end', disposeOnce)
+      stream.on('close', disposeOnce)
+      stream.on('error', disposeOnce)
+      info('everything is ready, will transfer', stream.length)
+      streams[`${vdiRef}.vhd`] = stream
+    }
+    return {
+      streams,
+      vbds,
+      vdis,
+      version: '1.0.0',
+      vifs,
+      vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+    }
+  }
+
+  async #decorateIncrementalVmMetadata() {
+    const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
+
+    const ignoredVdis = new Set(
+      Object.entries(mapVdisSrs)
+        .filter(([_, srUuid]) => srUuid === null)
+        .map(([vdiUuid]) => vdiUuid)
+    )
+    let backup
+    if (useDifferentialRestore) {
+      backup = await this._reuseNearestSnapshot(ignoredVdis)
+    } else {
+      backup = await this._adapter.readIncrementalVmBackup(this._metadata, ignoredVdis)
+    }
+    const xapi = this._xapi
+
+    const cache = new Map()
+    const mapVdisSrRefs = {}
+    if (additionnalVmTag !== undefined) {
+      backup.vm.tags.push(additionnalVmTag)
+    }
+    for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
+      mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
+    }
+    const srRef = await resolveUuid(xapi, cache, this._srUuid, 'SR')
+    Object.values(backup.vdis).forEach(vdi => {
+      vdi.SR = mapVdisSrRefs[vdi.uuid] ?? srRef
+    })
+    return backup
+  }
+
   async run() {
     const adapter = this._adapter
     const metadata = this._metadata
     const isFull = metadata.mode === 'full'
 
     const sizeContainer = { size: 0 }
-
+    const { newMacAddresses } = this._importIncrementalVmSettings
     let backup
     if (isFull) {
       backup = await adapter.readFullVmBackup(metadata)
@@ -28,12 +234,7 @@ export class ImportVmBackup {
     } else {
       assert.strictEqual(metadata.mode, 'delta')
 
-      const ignoredVdis = new Set(
-        Object.entries(this._importIncrementalVmSettings.mapVdisSrs)
-          .filter(([_, srUuid]) => srUuid === null)
-          .map(([vdiUuid]) => vdiUuid)
-      )
-      backup = await adapter.readIncrementalVmBackup(metadata, ignoredVdis)
+      backup = await this.#decorateIncrementalVmMetadata()
       Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
@@ -48,8 +249,7 @@ export class ImportVmBackup {
         const vmRef = isFull
           ? await xapi.VM_import(backup, srRef)
           : await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
-              ...this._importIncrementalVmSettings,
-              detectBase: false,
+              newMacAddresses,
             })
 
         await Promise.all([
@@ -59,6 +259,13 @@ export class ImportVmBackup {
             vmRef,
             `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
           ),
+          xapi.call(
+            'VM.set_name_description',
+            vmRef,
+            `Restored on ${formatFilenameDate(+new Date())} from ${adapter._handler._remote.name} -
+             ${metadata.vm.name_description}
+            `
+          ),
         ])
 
         return {
@@ -69,3 +276,5 @@ export class ImportVmBackup {
     )
   }
 }
+
+decorateClass(ImportVmBackup, { _reuseNearestSnapshot: defer })

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -35,6 +35,8 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
+const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
+
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -189,13 +191,14 @@ export class RemoteAdapter {
   // check if we will be allowed to merge a a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
-    return await Disposable.use(openVhd(this.handler, path), vhd => {
+    return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
       // this baseUuid is not linked with this vhd
       if (!vhd.footer.uuid.equals(packedParentUid)) {
         return false
       }
 
-      const isVhdDirectory = vhd instanceof VhdDirectory
+      // check if all the chain is composed of vhd directory
+      const isVhdDirectory = vhd.checkVhdsClass(VhdDirectory)
       return isVhdDirectory
         ? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
         : !this.useVhdDirectory()
@@ -681,11 +684,13 @@ export class RemoteAdapter {
     }
   }
 
-  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+  async outputStream(path, input, { checksum = true, maxStreamLength, streamLength, validator = noop } = {}) {
     const container = watchStreamSize(input)
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
+      maxStreamLength,
+      streamLength,
       async validator() {
         await input.task
         return validator.apply(this, arguments)
@@ -747,10 +752,37 @@ export class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
+    let json
+    let isImmutable = false
+    let remoteIsImmutable = false
+    // if the remote is immutable, check if this metadatas are also immutables
+    try {
+      // this file is not encrypted
+      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
+      remoteIsImmutable = true
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    try {
+      // this will trigger an EPERM error if the file is immutable
+      json = await this.handler.readFile(path, { flag: 'r+' })
+      // s3 handler don't respect flags
+    } catch (err) {
+      // retry without triggerring immutbaility check ,only on immutable remote
+      if (err.code === 'EPERM' && remoteIsImmutable) {
+        isImmutable = true
+        json = await this._handler.readFile(path, { flag: 'r' })
+      } else {
+        throw err
+      }
+    }
     // _filename is a private field used to compute the backup id
     //
     // it's enumerable to make it cacheable
-    const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+    const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
 
     // backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
     if (typeof metadata.vm.is_a_template === 'number') {

@xen-orchestra/backups/RestoreMetadataBackup.mjs

@@ -21,7 +21,7 @@ export class RestoreMetadataBackup {
       })
     } else {
       const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
-      const dataFileName = resolve(backupId, metadata.data ?? 'data.json')
+      const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
       const data = await handler.readFile(dataFileName)
 
       // if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it

@xen-orchestra/backups/_backupWorker.mjs

@@ -2,7 +2,10 @@ import { createLogger } from '@xen-orchestra/log'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure'
 
 import Disposable from 'promise-toolbox/Disposable'
+import humanFormat from 'human-format'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import mapValues from 'lodash/mapValues.js'
+import ms from 'ms'
 import { compose } from '@vates/compose'
 import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
@@ -20,7 +23,7 @@ createCachedLookup().patchGlobal()
 
 const logger = createLogger('xo:backups:worker')
 catchGlobalErrors(logger)
-const { debug } = logger
+const { debug, info, warn } = logger
 
 class BackupWorker {
   #config
@@ -149,6 +152,27 @@ process.on('message', async message => {
   debug('message received', { message })
 
   if (message.action === 'run') {
+    const resourceStart = process.resourceUsage()
+    const timeStart = process.hrtime.bigint()
+    info('starting backup')
+
+    process.on('exit', exitCode => {
+      const resourceUsage = mapValues(process.resourceUsage(), (end, key) => end - resourceStart[key])
+      const cpuTotal = resourceUsage.userCPUTime + resourceUsage.systemCPUTime
+      const duration = Number((process.hrtime.bigint() - timeStart) / 1000n) // in μs
+
+      info('process will exit', {
+        duration,
+        exitCode,
+        resourceUsage,
+        summary: {
+          duration: ms(duration / 1000),
+          cpuUsage: Math.round((100 * cpuTotal) / duration) + '%',
+          memoryUsage: humanFormat.bytes(resourceUsage.maxRSS * 1024),
+        },
+      })
+    })
+
     const backupWorker = new BackupWorker(message.data)
     try {
       const result = message.runWithLogs
@@ -177,8 +201,16 @@ process.on('message', async message => {
         status: 'failure',
       })
     } finally {
+      info('backup has ended')
       await ignoreErrors.call(backupWorker.debounceResource.flushAll())
       process.disconnect()
+
+      setTimeout(() => {
+        warn('worker process did not exit automatically, forcing...')
+
+        // eslint-disable-next-line n/no-process-exit
+        process.exit()
+      }, 30e3).unref()
     }
   }
 })

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -67,6 +67,11 @@ async function generateVhd(path, opts = {}) {
     await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
   }
 
+  if (opts.blocks) {
+    for (const blockId of opts.blocks) {
+      await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
+    }
+  }
   await vhd.writeBlockAllocationTable()
   await vhd.writeHeader()
   await vhd.writeFooter()
@@ -230,7 +235,7 @@ test('it merges delta of non destroyed chain', async () => {
 
   const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
   // size should be the size of children + grand children after the merge
-  assert.equal(metadata.size, 209920)
+  assert.equal(metadata.size, 104960)
 
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
   // only check deletion
@@ -320,6 +325,7 @@ describe('tests multiple combination ', () => {
         const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
           useAlias,
           mode: vhdMode,
+          blocks: [1, 3],
         })
         const child = await generateVhd(`${basePath}/child.vhd`, {
           useAlias,
@@ -328,6 +334,7 @@ describe('tests multiple combination ', () => {
             parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
             parentUuid: ancestor.footer.uuid,
           },
+          blocks: [1, 2],
         })
         // a grand child  vhd in metadata
         await generateVhd(`${basePath}/grandchild.vhd`, {
@@ -337,6 +344,7 @@ describe('tests multiple combination ', () => {
             parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
             parentUuid: child.footer.uuid,
           },
+          blocks: [2, 3],
         })
 
         // an older parent that was merging in clean
@@ -395,7 +403,7 @@ describe('tests multiple combination ', () => {
 
         const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
         // size should be the size of children + grand children + clean after the merge
-        assert.deepEqual(metadata.size, vhdMode === 'file' ? 314880 : undefined)
+        assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
 
         // broken vhd, non referenced, abandonned should be deleted ( alias and data)
         // ancestor and child should be merged

@xen-orchestra/backups/_cleanVm.mjs

@@ -3,7 +3,7 @@ import sum from 'lodash/sum.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
-import { dirname, resolve } from 'node:path'
+import { basename, dirname, resolve } from 'node:path'
 import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { mergeVhdChain } from 'vhd-lib/merge.js'
@@ -36,34 +36,32 @@ const computeVhdsSize = (handler, vhdPaths) =>
   )
 
 // chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
-  if (merge) {
-    logInfo(`merging VHD chain`, { chain })
+async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
+  logInfo(`merging VHD chain`, { chain })
 
-    let done, total
-    const handle = setInterval(() => {
-      if (done !== undefined) {
-        logInfo('merge in progress', {
-          done,
-          parent: chain[0],
-          progress: Math.round((100 * done) / total),
-          total,
-        })
-      }
-    }, 10e3)
-    try {
-      return await mergeVhdChain(handler, chain, {
-        logInfo,
-        mergeBlockConcurrency,
-        onProgress({ done: d, total: t }) {
-          done = d
-          total = t
-        },
-        removeUnused: remove,
+  let done, total
+  const handle = setInterval(() => {
+    if (done !== undefined) {
+      logInfo('merge in progress', {
+        done,
+        parent: chain[0],
+        progress: Math.round((100 * done) / total),
+        total,
       })
-    } finally {
-      clearInterval(handle)
     }
+  }, 10e3)
+  try {
+    return await mergeVhdChain(handler, chain, {
+      logInfo,
+      mergeBlockConcurrency,
+      onProgress({ done: d, total: t }) {
+        done = d
+        total = t
+      },
+      removeUnused: remove,
+    })
+  } finally {
+    clearInterval(handle)
   }
 }
 
@@ -181,7 +179,8 @@ export async function cleanVm(
   vmDir,
   {
     fixMetadata,
-    remove,
+    remove = false,
+    removeTmp = remove,
     merge,
     mergeBlockConcurrency,
     mergeLimiter = defaultMergeLimiter,
@@ -202,6 +201,10 @@ export async function cleanVm(
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
+    if(removeTmp && basename(path)[0] === '.'){
+      logInfo('deleting temporary VHD', { path })
+        return VhdAbstract.unlink(handler, path)
+    }
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
@@ -439,7 +442,8 @@ export async function cleanVm(
         }
       }
 
-      logWarn('unused VHD', { path: vhd })
+      // no warning because a VHD can be unused for perfectly good reasons,
+      // e.g. the corresponding backup (metadata file) has been deleted
       if (remove) {
         logInfo('deleting unused VHD', { path: vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
@@ -471,23 +475,20 @@ export async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(handler, chain, {
+      const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
         logInfo,
         logWarn,
         remove,
-        merge,
         mergeBlockConcurrency,
       })
-      if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
-        metadataWithMergedVhd[metadataPath] = true
-      }
+      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
+      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
     })
   }
 
   await Promise.all([
     ...unusedVhdsDeletion,
-    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : () => Promise.resolve()),
     asyncMap(unusedXvas, path => {
       logWarn('unused XVA', { path })
       if (remove) {
@@ -509,12 +510,11 @@ export async function cleanVm(
 
   // update size for delta metadata with merged VHD
   // check for the other that the size is the same as the real file size
-
   await asyncMap(jsons, async metadataPath => {
     const metadata = backups.get(metadataPath)
 
     let fileSystemSize
-    const merged = metadataWithMergedVhd[metadataPath] !== undefined
+    const mergedSize = metadataWithMergedVhd[metadataPath]
 
     const { mode, size, vhds, xva } = metadata
 
@@ -524,26 +524,29 @@ export async function cleanVm(
         const linkedXva = resolve('/', vmDir, xva)
         try {
           fileSystemSize = await handler.getSize(linkedXva)
+          if (fileSystemSize !== size && fileSystemSize !== undefined) {
+            logWarn('cleanVm: incorrect backup size in metadata', {
+              path: metadataPath,
+              actual: size ?? 'none',
+              expected: fileSystemSize,
+            })
+          }
         } catch (error) {
           // can fail with encrypted remote
         }
       } else if (mode === 'delta') {
-        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
-        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
-
-        // the size is not computed in some cases (e.g. VhdDirectory)
-        if (fileSystemSize === undefined) {
-          return
-        }
-
         // don't warn if the size has changed after a merge
-        if (!merged && fileSystemSize !== size) {
-          // FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
-          console.warn('cleanVm: incorrect backup size in metadata', {
-            path: metadataPath,
-            actual: size ?? 'none',
-            expected: fileSystemSize,
-          })
+        if (mergedSize === undefined) {
+          const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+          fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+          // the size is not computed in some cases (e.g. VhdDirectory)
+          if (fileSystemSize !== undefined && fileSystemSize !== size) {
+            logWarn('cleanVm: incorrect backup size in metadata', {
+              path: metadataPath,
+              actual: size ?? 'none',
+              expected: fileSystemSize,
+            })
+          }
         }
       }
     } catch (error) {
@@ -551,9 +554,19 @@ export async function cleanVm(
       return
     }
 
-    // systematically update size after a merge
-    if ((merged || fixMetadata) && size !== fileSystemSize) {
-      metadata.size = fileSystemSize
+    // systematically update size and differentials after a merge
+
+    // @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
+    if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
+      metadata.size = mergedSize ?? fileSystemSize ?? size
+
+      if (mergedSize) {
+        // all disks are now key disk
+        metadata.isVhdDifferencing = {}
+        for (const id of Object.values(metadata.vdis ?? {})) {
+          metadata.isVhdDifferencing[`${id}.vhd`] = false
+        }
+      }
       mustRegenerateCache = true
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })

@xen-orchestra/backups/_incrementalVm.mjs

@@ -1,7 +1,5 @@
-import find from 'lodash/find.js'
 import groupBy from 'lodash/groupBy.js'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
-import omit from 'lodash/omit.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { CancelToken } from 'promise-toolbox'
 import { compareVersions } from 'compare-versions'
@@ -11,48 +9,16 @@ import { defer } from 'golike-defer'
 import { cancelableMap } from './_cancelableMap.mjs'
 import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
-
-export const TAG_BASE_DELTA = 'xo:base_delta'
-
-export const TAG_COPY_SRC = 'xo:copy_of'
-
-const TAG_BACKUP_SR = 'xo:backup:sr'
+import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
-const resolveUuid = async (xapi, cache, uuid, type) => {
-  if (uuid == null) {
-    return uuid
-  }
-  let ref = cache.get(uuid)
-  if (ref === undefined) {
-    ref = await xapi.call(`${type}.get_by_uuid`, uuid)
-    cache.set(uuid, ref)
-  }
-  return ref
-}
 
 export async function exportIncrementalVm(
   vm,
-  baseVm,
-  {
-    cancelToken = CancelToken.none,
-
-    // Sets of UUIDs of VDIs that must be exported as full.
-    fullVdisRequired = new Set(),
-
-    disableBaseTags = false,
-    preferNbd,
-  } = {}
+  baseVdis = {},
+  { cancelToken = CancelToken.none, nbdConcurrency = 1, preferNbd } = {}
 ) {
   // refs of VM's VDIs → base's VDIs.
-  const baseVdis = {}
-  baseVm &&
-    baseVm.$VBDs.forEach(vbd => {
-      let vdi, snapshotOf
-      if ((vdi = vbd.$VDI) && (snapshotOf = vdi.$snapshot_of) && !fullVdisRequired.has(snapshotOf.uuid)) {
-        baseVdis[vdi.snapshot_of] = vdi
-      }
-    })
 
   const streams = {}
   const vdis = {}
@@ -73,13 +39,16 @@ export async function exportIncrementalVm(
     }
 
     // Look for a snapshot of this vdi in the base VM.
-    const baseVdi = baseVdis[vdi.snapshot_of]
+    const baseVdi = baseVdis[vdi.$snapshot_of.uuid]
 
     vdis[vdiRef] = {
       ...vdi,
       other_config: {
         ...vdi.other_config,
-        [TAG_BASE_DELTA]: baseVdi && !disableBaseTags ? baseVdi.uuid : undefined,
+        [BASE_DELTA_VDI]: baseVdi?.uuid,
+        [VM_UUID]:
+          vm.$snapshot_of?.uuid ?? // vm is a snapshot
+          vm.uuid, // vm is a not snapshot
       },
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
@@ -89,6 +58,7 @@ export async function exportIncrementalVm(
       baseRef: baseVdi?.$ref,
       cancelToken,
       format: 'vhd',
+      nbdConcurrency,
       preferNbd,
     })
   })
@@ -125,13 +95,6 @@ export async function exportIncrementalVm(
       vifs,
       vm: {
         ...vm,
-        other_config:
-          baseVm && !disableBaseTags
-            ? {
-                ...vm.other_config,
-                [TAG_BASE_DELTA]: baseVm.uuid,
-              }
-            : omit(vm.other_config, TAG_BASE_DELTA),
       },
     },
     'streams',
@@ -147,7 +110,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   $defer,
   incrementalVm,
   sr,
-  { cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
+  { cancelToken = CancelToken.none, newMacAddresses = false } = {}
 ) {
   const { version } = incrementalVm
   if (compareVersions(version, '1.0.0') < 0) {
@@ -157,35 +120,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   const vmRecord = incrementalVm.vm
   const xapi = sr.$xapi
 
-  let baseVm
-  if (detectBase) {
-    const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
-    if (remoteBaseVmUuid) {
-      baseVm = find(
-        xapi.objects.all,
-        obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
-      )
-
-      if (!baseVm) {
-        throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
-      }
-    }
-  }
-
-  const cache = new Map()
-  const mapVdisSrRefs = {}
-  for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
-    mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
-  }
-
-  const baseVdis = {}
-  baseVm &&
-    baseVm.$VBDs.forEach(vbd => {
-      const vdi = vbd.$VDI
-      if (vdi !== undefined) {
-        baseVdis[vbd.VDI] = vbd.$VDI
-      }
-    })
   const vdiRecords = incrementalVm.vdis
 
   // 0. Create suspend_VDI
@@ -197,18 +131,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
         vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
       })
     } else {
-      suspendVdi = await xapi.getRecord(
-        'VDI',
-        await xapi.VDI_create({
-          ...vdi,
-          other_config: {
-            ...vdi.other_config,
-            [TAG_BASE_DELTA]: undefined,
-            [TAG_COPY_SRC]: vdi.uuid,
-          },
-          sr: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
-        })
-      )
+      suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
       $defer.onFailure(() => suspendVdi.$destroy())
     }
   }
@@ -226,10 +149,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       ha_always_run: false,
       is_a_template: false,
       name_label: '[Importing…] ' + vmRecord.name_label,
-      other_config: {
-        ...vmRecord.other_config,
-        [TAG_COPY_SRC]: vmRecord.uuid,
-      },
     },
     {
       bios_strings: vmRecord.bios_strings,
@@ -250,17 +169,11 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     const vdi = vdiRecords[vdiRef]
     let newVdi
 
-    const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
-    if (remoteBaseVdiUuid) {
-      const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
-      if (!baseVdi) {
-        throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
-      }
-
-      newVdi = await xapi.getRecord('VDI', await baseVdi.$clone())
+    if (vdi.baseVdi !== undefined) {
+      newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
       $defer.onFailure(() => newVdi.$destroy())
 
-      await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
+      await newVdi.update_other_config(COPY_OF, vdi.uuid)
       if (vdi.virtual_size > newVdi.virtual_size) {
         await newVdi.$callAsync('resize', vdi.virtual_size)
       }
@@ -268,18 +181,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       // suspendVDI has already created
       newVdi = suspendVdi
     } else {
-      newVdi = await xapi.getRecord(
-        'VDI',
-        await xapi.VDI_create({
-          ...vdi,
-          other_config: {
-            ...vdi.other_config,
-            [TAG_BASE_DELTA]: undefined,
-            [TAG_COPY_SRC]: vdi.uuid,
-          },
-          SR: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
-        })
-      )
+      newVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
       $defer.onFailure(() => newVdi.$destroy())
     }
 
@@ -318,13 +220,19 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     // Import VDI contents.
     cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
       for (let stream of ensureArray(streams[`${id}.vhd`])) {
+        if (stream === null) {
+          // we restore a backup and reuse completly a local snapshot
+          continue
+        }
         if (typeof stream === 'function') {
           stream = await stream()
         }
         if (stream.length === undefined) {
           stream = await createVhdStreamWithLength(stream)
         }
+        await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
         await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
+        await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
       }
     }),
 

@xen-orchestra/backups/_otherConfig.mjs

@@ -0,0 +1,165 @@
+import { formatDateTime } from '@xen-orchestra/xapi'
+import assert from 'node:assert/strict'
+// in `other_config` of an incrementally replicated VM or VDI
+// contains the UUID of the object used as a base for an incremental export
+// used to search for the replica of the base before applying a incremental replication
+export const BASE_DELTA_VDI = 'xo:base_delta_vdi'
+
+// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
+//
+// added after the complete replication
+export const REPLICATED_TO_SR_UUID = 'xo:backup:sr'
+
+// in other_config of VDIs of an incrementally replicated VM
+// contains the UUID of the source  exported object (snapshot or VM)
+
+export const COPY_OF = 'xo:copy_of'
+
+export const DATETIME = 'xo:backup:datetime'
+
+export const JOB_ID = 'xo:backup:job'
+export const SCHEDULE_ID = 'xo:backup:schedule'
+
+// contains the number of delta in a chain, stored as a string
+export const DELTA_CHAIN_LENGTH = 'xo:backup:deltaChainLength'
+// contains the string true if this vdi has been exported successfully
+export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
+
+// the VM ( not the snapshot) uuid
+export const VM_UUID = 'xo:backup:vm'
+
+async function listVdiRefs(xapi, vmRef) {
+  return xapi.VM_getDisks(vmRef)
+}
+
+async function applyToVmAndVdis(xapi, vmRef, fn) {
+  const vdiRefs = await listVdiRefs(xapi, vmRef)
+  return Promise.all([fn('VM', vmRef), ...vdiRefs.map(vdiRef => fn('VDI', vdiRef))])
+}
+
+async function getDeltaChainLength(xapi, type, ref) {
+  const otherConfig = await xapi.getField(type, ref, 'other_config')
+  return Number(otherConfig[DELTA_CHAIN_LENGTH] ?? 0)
+}
+
+/**
+ * set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @param {Number} length
+ * @returns {Promise}
+ */
+export async function setVmDeltaChainLength(xapi, vmRef, length) {
+  return applyToVmAndVdis(xapi, vmRef, async (type, ref) => {
+    await xapi.setFieldEntry(type, ref, 'other_config', DELTA_CHAIN_LENGTH, String(length))
+  })
+}
+
+/**
+ * Compute the delta chain length of a VM and its associated VDIs
+ * if there is a discrependcy, use, the highest value
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function getVmDeltaChainLength(xapi, vmRef) {
+  const lengths = await applyToVmAndVdis(xapi, vmRef, async (type, ref) => getDeltaChainLength(xapi, type, ref))
+  return Math.max(...lengths)
+}
+
+/**
+ *
+ * Reset the other_config field of a VM and its VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export function resetVmOtherConfig(xapi, vmRef) {
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
+    return xapi.setFieldEntries(type, ref, 'other_config', {
+      [DATETIME]: null,
+      [DELTA_CHAIN_LENGTH]: null,
+      [EXPORTED_SUCCESSFULLY]: null,
+      [JOB_ID]: null,
+      [SCHEDULE_ID]: null,
+      [VM_UUID]: null,
+      // REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
+    })
+  })
+}
+
+/**
+ *
+ * used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function populateVdisOtherConfig(xapi, vmRef) {
+  const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
+  const {
+    [DATETIME]: datetime,
+    [DELTA_CHAIN_LENGTH]: chainLength,
+    [EXPORTED_SUCCESSFULLY]: successfully,
+    [JOB_ID]: jobId,
+    [REPLICATED_TO_SR_UUID]: replicatedTo,
+    [SCHEDULE_ID]: scheduleId,
+    [VM_UUID]: vmUuid,
+  } = otherConfig
+
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntries(type, ref, 'other_config', {
+      [DATETIME]: datetime,
+      [DELTA_CHAIN_LENGTH]: chainLength,
+      [EXPORTED_SUCCESSFULLY]: successfully,
+      [JOB_ID]: jobId,
+      [REPLICATED_TO_SR_UUID]: replicatedTo,
+      [SCHEDULE_ID]: scheduleId,
+      [VM_UUID]: vmUuid,
+    })
+  )
+}
+
+/**
+ *
+ * set the other_config key related to a backup of a VM and its associated VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @param {*} settings
+ * @returns {PRomise}
+ */
+export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, scheduleId, vmUuid, srUuid = null, ...other }) {
+  assert.notEqual(timestamp, undefined)
+  assert.notEqual(jobId, undefined)
+  assert.notEqual(scheduleId, undefined)
+  assert.notEqual(vmUuid, undefined)
+  // srUuid is nullish for backup
+  assert.equal(Object.keys(other).length, 0)
+
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntries(type, ref, 'other_config', {
+      [REPLICATED_TO_SR_UUID]: srUuid,
+      [DATETIME]: formatDateTime(timestamp),
+      [JOB_ID]: jobId,
+      [SCHEDULE_ID]: scheduleId,
+      [VM_UUID]: vmUuid,
+    })
+  )
+}
+/**
+ *
+ * mark the export of he VM and its VDIs as successfull
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function markExportSuccessfull(xapi, vmRef) {
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
+  )
+}

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -6,11 +6,12 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
 import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
-import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_REMOTE_VM_SETTINGS = {
   concurrency: 2,
   copyRetention: 0,
@@ -18,8 +19,10 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
   exportRetention: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
+  healthCheckTimeout: '10m',
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   timeout: 0,
   validateVhdStreams: false,
   vmTimeout: 0,
@@ -41,6 +44,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       () => this._getAdapter(job.sourceRemote),
       () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -62,8 +66,19 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmsUuids)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           const opts = {
             baseSettings,
@@ -72,7 +87,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             healthCheckSr,
             remoteAdapters,
             schedule,
-            settings: { ...settings, ...allSettings[vmUuid] },
+            settings: vmSettings,
             sourceRemoteAdapter,
             throttleStream,
             vmUuid,
@@ -86,10 +101,46 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
           }
 
-          return runTask(taskStart, () => vmBackup.run())
+          return sourceRemoteAdapter
+            .listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
+            .then(vmBackups => {
+              // avoiding to create tasks for empty directories
+              if (vmBackups.length > 0) {
+                if (taskByVmId[vmUuid] === undefined) {
+                  taskByVmId[vmUuid] = new Task(taskStart)
+                }
+                const task = taskByVmId[vmUuid]
+                return task
+                  .run(async () => {
+                    try {
+                      const result = await vmBackup.run()
+                      task.success(result)
+                      return result
+                    } catch (error) {
+                      if (isLastRun) {
+                        throw error
+                      } else {
+                        Task.warning(`Retry the VM mirror backup due to an error`, {
+                          attempt: nTriesByVmId[vmUuid],
+                          error: error.message,
+                        })
+                        queue.add(vmUuid)
+                      }
+                    }
+                  })
+                  .catch(noop)
+              }
+            })
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -11,8 +11,11 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_XAPI_VM_SETTINGS = {
   bypassVdiChainsCheck: false,
+  cbtDestroySnapshotData: false,
   checkpointSnapshot: false,
   concurrency: 2,
   copyRetention: 0,
@@ -22,8 +25,10 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   fullInterval: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
+  healthCheckTimeout: '10m',
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,
@@ -53,6 +58,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -89,48 +95,98 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmIds)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          const getVmTask = () => {
+            if (taskByVmId[vmUuid] === undefined) {
+              taskByVmId[vmUuid] = new Task(taskStart)
+            }
+            return taskByVmId[vmUuid]
+          }
+          const vmBackupFailed = error => {
+            if (isLastRun) {
+              throw error
+            } else {
+              Task.warning(`Retry the VM backup due to an error`, {
+                attempt: nTriesByVmId[vmUuid],
+                error: error.message,
+              })
+              queue.add(vmUuid)
+            }
+          }
+
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           return this._getRecord('VM', vmUuid).then(
             disposableVm =>
-              Disposable.use(disposableVm, vm => {
-                taskStart.data.name_label = vm.name_label
-                return runTask(taskStart, () => {
-                  const opts = {
-                    baseSettings,
-                    config,
-                    getSnapshotNameLabel,
-                    healthCheckSr,
-                    job,
-                    remoteAdapters,
-                    schedule,
-                    settings: { ...settings, ...allSettings[vm.uuid] },
-                    srs,
-                    throttleStream,
-                    vm,
-                  }
-                  let vmBackup
-                  if (job.mode === 'delta') {
-                    vmBackup = new IncrementalXapi(opts)
-                  } else {
-                    if (job.mode === 'full') {
-                      vmBackup = new FullXapi(opts)
-                    } else {
-                      throw new Error(`Job mode ${job.mode} not implemented`)
+              Disposable.use(disposableVm, async vm => {
+                if (taskStart.data.name_label === undefined) {
+                  taskStart.data.name_label = vm.name_label
+                }
+
+                const task = getVmTask()
+                return task
+                  .run(async () => {
+                    const opts = {
+                      baseSettings,
+                      config,
+                      getSnapshotNameLabel,
+                      healthCheckSr,
+                      job,
+                      remoteAdapters,
+                      schedule,
+                      settings: vmSettings,
+                      srs,
+                      throttleStream,
+                      vm,
                     }
-                  }
-                  return vmBackup.run()
-                })
+
+                    let vmBackup
+                    if (job.mode === 'delta') {
+                      vmBackup = new IncrementalXapi(opts)
+                    } else {
+                      if (job.mode === 'full') {
+                        vmBackup = new FullXapi(opts)
+                      } else {
+                        throw new Error(`Job mode ${job.mode} not implemented`)
+                      }
+                    }
+
+                    try {
+                      const result = await vmBackup.run()
+                      task.success(result)
+                      return result
+                    } catch (error) {
+                      vmBackupFailed(error)
+                    }
+                  })
+                  .catch(noop) // errors are handled by logs
               }),
             error =>
-              runTask(taskStart, () => {
-                throw error
+              getVmTask().run(() => {
+                vmBackupFailed(error)
               })
           )
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

@xen-orchestra/backups/_runners/_PoolMetadataBackup.mjs

@@ -31,7 +31,7 @@ export class PoolMetadataBackup {
     const poolDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${schedule.id}/${pool.$id}`
     const dir = `${poolDir}/${formatFilenameDate(timestamp)}`
 
-    const stream = await this._exportPoolMetadata()
+    const stream = (await this._exportPoolMetadata()).body
     const fileName = `${dir}/data`
 
     const metadata = JSON.stringify(

@xen-orchestra/backups/_runners/_forkStreamUnpipe.mjs

@@ -29,6 +29,10 @@ export function forkStreamUnpipe(source) {
     if (source.forks === 0) {
       debug('no more forks, destroying original stream')
       source.destroy(new Error('no more consumers for this stream'))
+    } else {
+      // a combination of stream.unpipe, onReadable and onData may stall stream here
+      // force it to flow again since we're piping it
+      source.resume()
     }
   })
   return fork

@xen-orchestra/backups/_runners/_vmRunners/FullRemote.mjs

@@ -1,5 +1,3 @@
-import { decorateMethodsWith } from '@vates/decorate-with'
-import { defer } from 'golike-defer'
 import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
@@ -10,15 +8,9 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
   _getRemoteWriter() {
     return FullRemoteWriter
   }
-  async _run($defer) {
+  async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
-    await this._callWriters(async writer => {
-      await writer.beforeBackup()
-      $defer(async () => {
-        await writer.afterBackup()
-      })
-    }, 'writer.beforeBackup()')
     if (transferList.length > 0) {
       for (const metadata of transferList) {
         const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
@@ -29,6 +21,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
           writer =>
             writer.run({
               stream: forkStreamUnpipe(stream),
+              // stream will be forked and transformed, it's not safe to attach additionnal properties to it
+              streamLength: stream.length,
               timestamp: metadata.timestamp,
               vm: metadata.vm,
               vmSnapshot: metadata.vmSnapshot,
@@ -44,7 +38,3 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
     }
   }
 }
-
-decorateMethodsWith(FullRemote, {
-  _run: defer,
-})

@xen-orchestra/backups/_runners/_vmRunners/FullXapi.mjs

@@ -30,18 +30,32 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
     const vm = this._vm
     const exportedVm = this._exportedVm
     const stream = this._throttleStream(
-      await this._xapi.VM_export(exportedVm.$ref, {
-        compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
-        useSnapshot: false,
-      })
+      (
+        await this._xapi.VM_export(exportedVm.$ref, {
+          compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
+          useSnapshot: false,
+        })
+      ).body
     )
+
+    const vdis = await exportedVm.$getDisks()
+    let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe
+    for (const vdiRef of vdis) {
+      const vdi = await this._xapi.getRecord('VDI', vdiRef)
+
+      // the size a of fully allocated vdi will be virtual_size  exaclty, it's a gross over evaluation
+      // of the real stream size in general, since a disk is never completly full
+      // vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
+      maxStreamLength += vdi.virtual_size
+    }
+
     const sizeContainer = watchStreamSize(stream)
 
     const timestamp = Date.now()
-
     await this._callWriters(
       writer =>
         writer.run({
+          maxStreamLength,
           sizeContainer,
           stream: forkStreamUnpipe(stream),
           timestamp,

@xen-orchestra/backups/_runners/_vmRunners/IncrementalRemote.mjs

@@ -1,7 +1,6 @@
 import { asyncEach } from '@vates/async-each'
-import { decorateMethodsWith } from '@vates/decorate-with'
-import { defer } from 'golike-defer'
 import assert from 'node:assert'
+import * as UUID from 'uuid'
 import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import mapValues from 'lodash/mapValues.js'
 
@@ -9,33 +8,64 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { Task } from '../../Task.mjs'
+import { Disposable } from 'promise-toolbox'
+import { openVhd } from 'vhd-lib'
+import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return IncrementalRemoteWriter
   }
-  async _run($defer) {
+  async _selectBaseVm(metadata) {
+    // for each disk , get the parent
+    const baseUuidToSrcVdi = new Map()
+
+    // no previous backup for a base( =key) backup
+    if (metadata.isBase) {
+      return
+    }
+    await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
+      const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
+      if (isDifferencing) {
+        const vmDir = getVmBackupDir(metadata.vm.uuid)
+        const path = `${vmDir}/${metadata.vhds[id]}`
+        // don't catch error : we can't recover if the source vhd are missing
+        await Disposable.use(openVhd(this._sourceRemoteAdapter._handler, path), vhd => {
+          baseUuidToSrcVdi.set(UUID.stringify(vhd.header.parentUuid), vdi.$snapshot_of$uuid)
+        })
+      }
+    })
+
+    const presentBaseVdis = new Map(baseUuidToSrcVdi)
+    await this._callWriters(
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
+      'writer.checkBaseVdis()',
+      false
+    )
+    // check if the parent vdi are present in all the remotes
+    baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
+      if (!presentBaseVdis.has(baseUuid)) {
+        throw new Error(`Missing vdi ${baseUuid} which is a base for a delta`)
+      }
+    })
+    // yeah , let's go
+  }
+  async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
-    await this._callWriters(async writer => {
-      await writer.beforeBackup()
-      $defer(async () => {
-        await writer.afterBackup()
-      })
-    }, 'writer.beforeBackup()')
 
     if (transferList.length > 0) {
       for (const metadata of transferList) {
         assert.strictEqual(metadata.mode, 'delta')
-
+        await this._selectBaseVm(metadata)
         await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
         const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
           useChain: false,
         })
 
-        const differentialVhds = {}
+        const isVhdDifferencing = {}
 
         await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-          differentialVhds[key] = await isVhdDifferencingDisk(stream)
+          isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
         })
 
         incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
@@ -43,13 +73,24 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
           writer =>
             writer.transfer({
               deltaExport: forkDeltaExport(incrementalExport),
-              differentialVhds,
+              isVhdDifferencing,
               timestamp: metadata.timestamp,
               vm: metadata.vm,
               vmSnapshot: metadata.vmSnapshot,
             }),
           'writer.transfer()'
         )
+        // this will update parent name with the needed alias
+        await this._callWriters(
+          writer =>
+            writer.updateUuidAndChain({
+              isVhdDifferencing,
+              timestamp: metadata.timestamp,
+              vdis: incrementalExport.vdis,
+            }),
+          'writer.updateUuidAndChain()'
+        )
+
         await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
         // for healthcheck
         this._tags = metadata.vm.tags
@@ -61,6 +102,3 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
 }
 
 export const IncrementalRemote = IncrementalRemoteVmBackupRunner
-decorateMethodsWith(IncrementalRemoteVmBackupRunner, {
-  _run: defer,
-})

@xen-orchestra/backups/_runners/_vmRunners/IncrementalXapi.mjs

@@ -1,8 +1,6 @@
 import { asyncEach } from '@vates/async-each'
-import { asyncMap } from '@xen-orchestra/async-map'
 import { createLogger } from '@xen-orchestra/log'
 import { pipeline } from 'node:stream'
-import findLast from 'lodash/findLast.js'
 import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import keyBy from 'lodash/keyBy.js'
 import mapValues from 'lodash/mapValues.js'
@@ -15,9 +13,15 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
 import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
 import { Task } from '../../Task.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import {
+  DATETIME,
+  DELTA_CHAIN_LENGTH,
+  EXPORTED_SUCCESSFULLY,
+  setVmDeltaChainLength,
+  markExportSuccessfull,
+} from '../../_otherConfig.mjs'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
-
 const noop = Function.prototype
 
 export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
@@ -30,17 +34,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
   }
 
   async _copy() {
-    const baseVm = this._baseVm
+    const baseVdis = this._baseVdis
     const vm = this._vm
     const exportedVm = this._exportedVm
-    const fullVdisRequired = this._fullVdisRequired
-
-    const isFull = fullVdisRequired === undefined || fullVdisRequired.size !== 0
 
+    const isFull = Object.values(baseVdis).length === 0 || Object.values(baseVdis).some(_ => !_)
     await this._callWriters(writer => writer.prepare({ isFull }), 'writer.prepare()')
 
-    const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
-      fullVdisRequired,
+    const deltaExport = await exportIncrementalVm(exportedVm, baseVdis, {
+      nbdConcurrency: this._settings.nbdConcurrency,
       preferNbd: this._settings.preferNbd,
     })
     // since NBD is network based, if one disk use nbd , all the disk use them
@@ -49,11 +51,11 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       Task.info('Transfer data using NBD')
     }
 
-    const differentialVhds = {}
+    const isVhdDifferencing = {}
     // since isVhdDifferencingDisk is reading and unshifting data in stream
     // it should be done BEFORE any other stream transform
     await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
-      differentialVhds[key] = await isVhdDifferencingDisk(stream)
+      isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
     })
     const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
 
@@ -68,7 +70,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       writer =>
         writer.transfer({
           deltaExport: forkDeltaExport(deltaExport),
-          differentialVhds,
+          isVhdDifferencing,
           sizeContainers,
           timestamp,
           vm,
@@ -77,18 +79,25 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       'writer.transfer()'
     )
 
-    this._baseVm = exportedVm
+    // we want to control the uuid of the vhd in the chain
+    // and ensure they are correctly chained
+    await this._callWriters(
+      writer =>
+        writer.updateUuidAndChain({
+          isVhdDifferencing,
+          timestamp,
+          vdis: deltaExport.vdis,
+        }),
+      'writer.updateUuidAndChain()'
+    )
 
-    if (baseVm !== undefined) {
-      await exportedVm.update_other_config(
-        'xo:backup:deltaChainLength',
-        String(+(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1)
-      )
-    }
+    if (!isFull) {
+      await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
+    } // on a full the delta chain will be null
 
     // not the case if offlineBackup
     if (exportedVm.is_a_snapshot) {
-      await exportedVm.update_other_config('xo:backup:exported', 'true')
+      await markExportSuccessfull(this._xapi, exportedVm.$ref)
     }
 
     const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
@@ -106,43 +115,67 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
   async _selectBaseVm() {
     const xapi = this._xapi
 
-    let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
-    if (baseVm === undefined) {
-      debug('no base VM found')
-      return
+    // filter _jobSnapshotVdis to have only the last successfully exported vdi
+    // compute delta chain length
+    // fill baseUuidToSrcVdi with this data
+
+    const exportedVdis = this._jobSnapshotVdis.filter(
+      _ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
+    )
+    let lastSuccessfullBackup
+    let lastExportedVdis = []
+    for (const exportedVdi of exportedVdis) {
+      if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
+        lastExportedVdis = []
+        lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
+      }
+      if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
+        lastExportedVdis.push(exportedVdi)
+      }
     }
 
+    this._baseVdis = {}
+    if (lastExportedVdis.length === 0) {
+      debug('no base VDIS found', {
+        jobLength: this._jobSnapshotVdis.length,
+        lastSuccessfullBackup,
+        exportedLength: exportedVdis.length,
+      })
+      return
+    }
+    const deltaChainLength = Math.max(
+      ...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
+    )
     const fullInterval = this._settings.fullInterval
-    const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
-    if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
-      debug('not using base VM becaust fullInterval reached')
+    if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
+      debug('not using base VM because fullInterval reached', {
+        fullInterval,
+        deltaChainLength,
+        eq: fullInterval < deltaChainLength + 1,
+        dc1: deltaChainLength + 1,
+      })
       return
     }
 
     const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
 
-    // resolve full record
-    baseVm = await xapi.getRecord('VM', baseVm.$ref)
-
-    const baseUuidToSrcVdi = new Map()
-    await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const [baseUuid, snapshotOf] = await Promise.all([
-        xapi.getField('VDI', baseRef, 'uuid'),
-        xapi.getField('VDI', baseRef, 'snapshot_of'),
-      ])
+    const baseUuidToSrcVdiUuid = new Map()
+    for (const lastExportedVdi of lastExportedVdis) {
+      const baseUuid = lastExportedVdi.uuid
+      const snapshotOf = lastExportedVdi.snapshot_of
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(baseUuid, srcVdi)
+        baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
       } else {
         debug('ignore snapshot VDI because no longer present on VM', {
           vdi: baseUuid,
         })
       }
-    })
+    }
 
-    const presentBaseVdis = new Map(baseUuidToSrcVdi)
+    const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
     await this._callWriters(
-      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseVm),
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
       'writer.checkBaseVdis()',
       false
     )
@@ -151,24 +184,22 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       debug('no base VM found')
       return
     }
-
-    const fullVdisRequired = new Set()
-    baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
+    baseUuidToSrcVdiUuid.forEach((srcVdiUuid, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {
         debug('found base VDI', {
           base: baseUuid,
-          vdi: srcVdi.uuid,
+          vdi: srcVdiUuid,
         })
+        this._baseVdis[srcVdiUuid] = lastExportedVdis.find(vdi => vdi.uuid === baseUuid)
       } else {
         debug('missing base VDI', {
           base: baseUuid,
-          vdi: srcVdi.uuid,
+          vdi: srcVdiUuid,
         })
-        fullVdisRequired.add(srcVdi.uuid)
+        // this will force a full for this vdi
+        this._baseVdis[srcVdiUuid] = undefined
       }
     })
-
-    this._baseVm = baseVm
-    this._fullVdisRequired = fullVdisRequired
+    this._deltaChainLength = deltaChainLength
   }
 }

@xen-orchestra/backups/_runners/_vmRunners/_AbstractRemote.mjs

@@ -1,4 +1,6 @@
 import { asyncEach } from '@vates/async-each'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
 import { Disposable } from 'promise-toolbox'
 
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
@@ -22,6 +24,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
+    this._settings = settings
     this.scheduleId = schedule.id
     this.timestamp = undefined
 
@@ -89,11 +92,21 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     return chain
   }
 
-  async run() {
+  async run($defer) {
     const handler = this._sourceRemoteAdapter._handler
     await Disposable.use(await handler.lock(getVmBackupDir(this._vmUuid)), async () => {
+      await this._callWriters(async writer => {
+        await writer.beforeBackup()
+        $defer(async () => {
+          await writer.afterBackup()
+        })
+      }, 'writer.beforeBackup()')
       await this._run()
       await this._healthCheck()
     })
   }
 }
+
+decorateMethodsWith(AbstractRemote, {
+  run: defer,
+})

@xen-orchestra/backups/_runners/_vmRunners/_AbstractXapi.mjs

@@ -4,11 +4,18 @@ import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
-import { formatDateTime } from '@xen-orchestra/xapi'
 
 import { getOldEntries } from '../../_getOldEntries.mjs'
 import { Task } from '../../Task.mjs'
 import { Abstract } from './_Abstract.mjs'
+import {
+  DATETIME,
+  JOB_ID,
+  SCHEDULE_ID,
+  populateVdisOtherConfig,
+  resetVmOtherConfig,
+  setVmOtherConfig,
+} from '../../_otherConfig.mjs'
 
 export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   constructor({
@@ -25,7 +32,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     vm,
   }) {
     super()
-    if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
+    if (vm.other_config[JOB_ID] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
       // will be replicated again and again
       throw new Error('cannot backup a VM created by this very job')
@@ -49,17 +56,17 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     this._exportedVm = undefined
     this._vm = vm
 
-    this._fullVdisRequired = undefined
+    this._baseVdis = undefined
     this._getSnapshotNameLabel = getSnapshotNameLabel
     this._isIncremental = job.mode === 'delta'
     this._healthCheckSr = healthCheckSr
     this._jobId = job.id
-    this._jobSnapshots = undefined
+    this._jobSnapshotVdis = undefined
     this._throttleStream = throttleStream
     this._xapi = vm.$xapi
 
     // Base VM for the export
-    this._baseVm = undefined
+    this._baseVdis = undefined
 
     // Settings for this specific run (job, schedule, VM)
     if (tags.includes('xo-memory-backup')) {
@@ -123,15 +130,8 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   // copied on manual snapshots and interfere with the backup jobs
   async _cleanMetadata() {
     const vm = this._vm
-    if ('xo:backup:job' in vm.other_config) {
-      await vm.update_other_config({
-        'xo:backup:datetime': null,
-        'xo:backup:deltaChainLength': null,
-        'xo:backup:exported': null,
-        'xo:backup:job': null,
-        'xo:backup:schedule': null,
-        'xo:backup:vm': null,
-      })
+    if (JOB_ID in vm.other_config) {
+      await resetVmOtherConfig(this._xapi, vm.$ref)
     }
   }
 
@@ -146,23 +146,29 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         if (!settings.bypassVdiChainsCheck) {
           await vm.$assertHealthyVdiChains()
         }
+        if (settings.preferNbd) {
+          try {
+            // enable CBT on all disks if possible
+            const diskRefs = await xapi.VM_getDisks(vm.$ref)
+            await Promise.all(diskRefs.map(diskRef => xapi.call('VDI.enable_cbt', diskRef)))
+          } catch (error) {
+            Task.info(`couldn't enable CBT`, error)
+          }
+        }
 
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
-          ignoreNobakVdis: true,
+          ignoredVdisTag: '[NOBAK]',
           name_label: this._getSnapshotNameLabel(vm),
           unplugVusbs: true,
         })
         this.timestamp = Date.now()
-
-        await xapi.setFieldEntries('VM', snapshotRef, 'other_config', {
-          'xo:backup:datetime': formatDateTime(this.timestamp),
-          'xo:backup:job': this._jobId,
-          'xo:backup:schedule': this.scheduleId,
-          'xo:backup:vm': vm.uuid,
+        await setVmOtherConfig(xapi, snapshotRef, {
+          timestamp: this.timestamp,
+          jobId: this._jobId,
+          scheduleId: this.scheduleId,
+          vmUuid: vm.uuid,
         })
-
         this._exportedVm = await xapi.getRecord('VM', snapshotRef)
-
         return this._exportedVm.uuid
       })
     } else {
@@ -176,40 +182,120 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     const vmRef = this._vm.$ref
     const xapi = this._xapi
 
-    const snapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
-    const snapshotsOtherConfig = await asyncMap(snapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
+    // to ensure compatibility with snapshots older than CBT implementation
+    // update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
+    // remove by then end of 2024
+    const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
+    const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
 
-    const snapshots = []
-    snapshotsOtherConfig.forEach((other_config, i) => {
-      if (other_config['xo:backup:job'] === jobId) {
-        snapshots.push({ other_config, $ref: snapshotsRef[i] })
+    const vmSnapshots = []
+    vmSnapshotsOtherConfig.forEach((other_config, i) => {
+      if (other_config[JOB_ID] === jobId) {
+        vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
       }
     })
-    snapshots.sort((a, b) => (a.other_config['xo:backup:datetime'] < b.other_config['xo:backup:datetime'] ? -1 : 1))
-    this._jobSnapshots = snapshots
+    await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
+    // end of compatibiliy handling
+
+    // handle snapshot by VDI
+    this._jobSnapshotVdis = []
+    const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
+    for (const srcVdi of srcVdis) {
+      const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
+      for (const snapshot of snapshots) {
+        if (snapshot.other_config[JOB_ID] === jobId) {
+          this._jobSnapshotVdis.push(snapshot)
+        }
+      }
+    }
   }
 
   async _removeUnusedSnapshots() {
     const allSettings = this.job.settings
     const baseSettings = this._baseSettings
-    const baseVmRef = this._baseVm?.$ref
 
-    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
+    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     const xapi = this._xapi
-    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+    await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
+      const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
+
+      const datetimes = Object.keys(snapshotPerDatetime)
+      datetimes.sort()
+
       const settings = {
         ...baseSettings,
         ...allSettings[scheduleId],
         ...allSettings[this._vm.uuid],
       }
-      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
-        if ($ref !== baseVmRef) {
-          return xapi.VM_destroy($ref)
+      // ensure we never delete the last one for delta
+      const minRetention = this.job.mode === 'delta' ? 1 : 0
+      const retention = Math.max(settings.snapshotRetention ?? 0, minRetention)
+      await asyncMap(getOldEntries(retention, datetimes), async datetime => {
+        const vdis = snapshotPerDatetime[datetime]
+        let vmRef
+        // if there is an attached VM => destroy the VM (Non CBT backups)
+        for (const vdi of vdis) {
+          if (vdi.$VBDs.length > 0) {
+            const vbds = vdi.$VBDs
+            // only one VM linked to this vdi
+            // this will throw error for VDI still attached to control domain
+            assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
+            const vm = vbds[0].$VM
+            assert.strictEqual(vm.is_control_domain, false, `Disk is still attached to DOM0 VM`) // don't delete a VM (especially a control domain)
+            assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
+
+            const vmRefVdi = vm.$ref
+            // same vm than other vdi of the same batch
+            assert.ok(
+              vmRef === undefined || vmRef === vmRefVdi,
+              '_removeUnusedSnapshots don t handle vdi related to multiple VMs '
+            )
+            vmRef = vmRefVdi
+          }
+        }
+        if (vmRef !== undefined) {
+          return xapi.VM_destroy(vmRef)
+        } else {
+          return asyncMap(
+            vdis.map(async ({ $ref }) => {
+              await xapi.VDI_destroy($ref)
+            })
+          )
         }
       })
     })
   }
 
+  async _removeSnapshotData() {
+    // now that we use CBT, we can destroy the data of the snapshot used for this backup
+    // going back to a previous version of XO not supporting CBT will create a full backup
+    // this will only do something after snapshot and transfer
+    if (
+      // don't modify the VM
+      this._exportedVm?.is_a_snapshot &&
+      // user don't want to keep the snapshot data
+      this._settings.snapshotRetention === 0 &&
+      // preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
+      // in that case next runs will be full, but there is not an easy way to prevent that
+      this._settings.preferNbd &&
+      // only delete snapshost data if the config allows it
+      this._settings.cbtDestroySnapshotData
+    ) {
+      Task.info('will delete snapshot data')
+      const vdiRefs = await this._xapi.VM_getDisks(this._exportedVm.$ref)
+      await this._xapi.call('VM.destroy', this._exportedVm.$ref)
+      for (const vdiRef of vdiRefs) {
+        try {
+          // data_destroy will fail with a VDI_NO_CBT_METADATA error if CBT is not enabled on this VDI
+          await this._xapi.call('VDI.data_destroy', vdiRef)
+          Task.info(`Snapshot data has been deleted`, { vdiRef })
+        } catch (error) {
+          Task.warning(`Couldn't deleted snapshot data`, { error, vdiRef })
+        }
+      }
+    }
+  }
+
   async copy() {
     throw new Error('Not implemented')
   }
@@ -240,6 +326,26 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       })
     }, 'writer.beforeBackup()')
 
+    const vm = this._vm
+
+    // block migration during the backup on the VM itself, not the latest snapshot
+    {
+      const { pool_migrate, migrate_send } = vm.blocked_operations
+
+      const reason = 'VM migration is blocked during backup'
+      await vm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
+
+      $defer(() =>
+        // delete the entries if they did not exist previously or if they were
+        // equal to reason (which happen if a previous backup was interrupted
+        // before resetting them)
+        vm.update_blocked_operations({
+          migrate_send: migrate_send === undefined || migrate_send === reason ? null : migrate_send,
+          pool_migrate: pool_migrate === undefined || pool_migrate === reason ? null : pool_migrate,
+        })
+      )
+    }
+
     await this._fetchJobSnapshots()
 
     await this._selectBaseVm()
@@ -247,7 +353,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     await this._cleanMetadata()
     await this._removeUnusedSnapshots()
 
-    const vm = this._vm
     const isRunning = vm.power_state === 'Running'
     const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
     if (startAfter) {
@@ -261,15 +366,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       }
 
       if (this._writers.size !== 0) {
-        const { pool_migrate = null, migrate_send = null } = this._exportedVm.blocked_operations
-
-        const reason = 'VM migration is blocked during backup'
-        await this._exportedVm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
-        try {
-          await this._copy()
-        } finally {
-          await this._exportedVm.update_blocked_operations({ pool_migrate, migrate_send })
-        }
+        await this._copy()
       }
     } finally {
       if (startAfter) {
@@ -278,6 +375,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
 
       await this._fetchJobSnapshots()
       await this._removeUnusedSnapshots()
+      await this._removeSnapshotData()
     }
     await this._healthCheck()
   }

@xen-orchestra/backups/_runners/_vmRunners/_forkDeltaExport.mjs

@@ -1,11 +1,11 @@
+import cloneDeep from 'lodash/cloneDeep.js'
 import mapValues from 'lodash/mapValues.js'
 
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 
 export function forkDeltaExport(deltaExport) {
-  return Object.create(deltaExport, {
-    streams: {
-      value: mapValues(deltaExport.streams, forkStreamUnpipe),
-    },
-  })
+  const { streams, ...rest } = deltaExport
+  const newMetadata = cloneDeep(rest)
+  newMetadata.streams = mapValues(streams, forkStreamUnpipe)
+  return newMetadata
 }

@xen-orchestra/backups/_runners/_writers/FullRemoteWriter.mjs

@@ -24,7 +24,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
     )
   }
 
-  async _run({ timestamp, sizeContainer, stream, vm, vmSnapshot }) {
+  async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
     const settings = this._settings
     const job = this._job
     const scheduleId = this._scheduleId
@@ -65,6 +65,8 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
 
     await Task.run({ name: 'transfer' }, async () => {
       await adapter.outputStream(dataFilename, stream, {
+        maxStreamLength,
+        streamLength,
         validator: tmpPath => adapter.isValidXva(tmpPath),
       })
       return { size: sizeContainer.size }

@xen-orchestra/backups/_runners/_writers/FullXapiWriter.mjs

@@ -1,6 +1,5 @@
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
-import { formatDateTime } from '@xen-orchestra/xapi'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -9,6 +8,7 @@ import { Task } from '../../Task.mjs'
 import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
 import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
 import { listReplicatedVms } from './_listReplicatedVms.mjs'
+import { setVmOtherConfig } from '../../_otherConfig.mjs'
 
 export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
   constructor(props) {
@@ -76,14 +76,12 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
           'Start operation for this vm is blocked, clone it if you want to use it.'
         )
       ),
-      targetVm.update_other_config({
-        'xo:backup:sr': srUuid,
-
-        // these entries need to be added in case of offline backup
-        'xo:backup:datetime': formatDateTime(timestamp),
-        'xo:backup:job': job.id,
-        'xo:backup:schedule': scheduleId,
-        'xo:backup:vm': vm.uuid,
+      setVmOtherConfig(xapi, targetVmRef, {
+        timestamp,
+        jobId: job.id,
+        scheduleId,
+        srUuid,
+        vmUuid: vm.uuid,
       }),
     ])
 

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -1,13 +1,12 @@
 import assert from 'node:assert'
 import mapValues from 'lodash/mapValues.js'
-import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncEach } from '@vates/async-each'
 import { asyncMap } from '@xen-orchestra/async-map'
-import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
+import { chainVhd, openVhd } from 'vhd-lib'
 import { createLogger } from '@xen-orchestra/log'
 import { decorateClass } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
-import { dirname } from 'node:path'
+import { dirname, basename } from 'node:path'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -22,42 +21,45 @@ import { Disposable } from 'promise-toolbox'
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
 export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
+  #parentVdiPaths
+  #vhds
   async checkBaseVdis(baseUuidToSrcVdi) {
+    this.#parentVdiPaths = {}
     const { handler } = this._adapter
     const adapter = this._adapter
 
     const vdisDir = `${this._vmBackupDir}/vdis/${this._job.id}`
 
-    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
-      let found = false
+    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdiUuid]) => {
+      let parentDestPath
+      const vhdDir = `${vdisDir}/${srcVdiUuid}`
       try {
-        const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
+        const vhds = await handler.list(vhdDir, {
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           ignoreMissing: true,
           prependDir: true,
         })
         const packedBaseUuid = packUuid(baseUuid)
-        await asyncMap(vhds, async path => {
-          try {
-            await checkVhdChain(handler, path)
-            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
-            //
-            // since all the checks of a path are done in parallel, found would be containing
-            // only the last answer of isMergeableParent which is probably not the right one
-            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
+        // the last one is probably the right one
 
-            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
-            found = found || isMergeable
+        for (let i = vhds.length - 1; i >= 0 && parentDestPath === undefined; i--) {
+          const path = vhds[i]
+          try {
+            if (await adapter.isMergeableParent(packedBaseUuid, path)) {
+              parentDestPath = path
+            }
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
-        })
+        }
       } catch (error) {
         warn('checkBaseVdis', { error })
       }
-      if (!found) {
+      // no usable parent => the runner will have to decide to fall back to a full or stop backup
+      if (parentDestPath === undefined) {
         baseUuidToSrcVdi.delete(baseUuid)
+      } else {
+        this.#parentVdiPaths[vhdDir] = parentDestPath
       }
     })
   }
@@ -122,6 +124,45 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     }
   }
 
+  async updateUuidAndChain({ isVhdDifferencing, vdis }) {
+    assert.notStrictEqual(
+      this.#vhds,
+      undefined,
+      '_transfer must be called before updateUuidAndChain for incremental backups'
+    )
+
+    const parentVdiPaths = this.#parentVdiPaths
+    const { handler } = this._adapter
+    const vhds = this.#vhds
+    await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
+      const isDifferencing = isVhdDifferencing[`${id}.vhd`]
+      const path = `${this._vmBackupDir}/${vhds[id]}`
+      if (isDifferencing) {
+        assert.notStrictEqual(
+          parentVdiPaths,
+          undefined,
+          'checkbasevdi must be called before updateUuidAndChain for incremental backups'
+        )
+        const parentPath = parentVdiPaths[dirname(path)]
+        // we are in a incremental backup
+        // we already computed the chain in checkBaseVdis
+        assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
+        // forbid any kind of loop
+        assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
+        await chainVhd(handler, parentPath, handler, path)
+      }
+
+      // set the correct UUID in the VHD if needed
+      await Disposable.use(openVhd(handler, path), async vhd => {
+        if (!vhd.footer.uuid.equals(packUuid(vdi.uuid))) {
+          vhd.footer.uuid = packUuid(vdi.uuid)
+          await vhd.readBlockAllocationTable() // required by writeFooter()
+          await vhd.writeFooter()
+        }
+      })
+    })
+  }
+
   async _deleteOldEntries() {
     const adapter = this._adapter
     const oldEntries = this._oldEntries
@@ -132,7 +173,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     }
   }
 
-  async _transfer($defer, { differentialVhds, timestamp, deltaExport, vm, vmSnapshot }) {
+  async _transfer($defer, { isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
     const adapter = this._adapter
     const job = this._job
     const scheduleId = this._scheduleId
@@ -140,14 +181,10 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     const jobId = job.id
     const handler = adapter.handler
 
-    let metadataContent = await this._isAlreadyTransferred(timestamp)
-    if (metadataContent !== undefined) {
-      // @todo : should skip backup while being vigilant to not stuck the forked stream
-      Task.info('This backup has already been transfered')
-    }
-
     const basename = formatFilenameDate(timestamp)
-    const vhds = mapValues(
+    // update this.#vhds before eventually skipping transfer, so that
+    // updateUuidAndChain has all the mandatory data
+    const vhds = (this.#vhds = mapValues(
       deltaExport.vdis,
       vdi =>
         `vdis/${jobId}/${
@@ -157,9 +194,18 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
               vdi.uuid
             : vdi.$snapshot_of$uuid
         }/${adapter.getVhdFileName(basename)}`
-    )
+    ))
+
+    let metadataContent = await this._isAlreadyTransferred(timestamp)
+    if (metadataContent !== undefined) {
+      // skip backup while being vigilant to not stuck the forked stream
+      Task.info('This backup has already been transfered')
+      Object.values(deltaExport.streams).forEach(stream => stream.destroy())
+      return { size: 0 }
+    }
 
     metadataContent = {
+      isVhdDifferencing,
       jobId,
       mode: job.mode,
       scheduleId,
@@ -172,56 +218,24 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       vm,
       vmSnapshot,
     }
+
     const { size } = await Task.run({ name: 'transfer' }, async () => {
       let transferSize = 0
       await asyncEach(
-        Object.entries(deltaExport.vdis),
-        async ([id, vdi]) => {
+        Object.keys(deltaExport.vdis),
+        async id => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
-
-          const isDelta = differentialVhds[`${id}.vhd`]
-          let parentPath
-          if (isDelta) {
-            const vdiDir = dirname(path)
-            parentPath = (
-              await handler.list(vdiDir, {
-                filter: filename => filename[0] !== '.' && filename.endsWith('.vhd'),
-                prependDir: true,
-              })
-            )
-              .sort()
-              .pop()
-
-            assert.notStrictEqual(
-              parentPath,
-              undefined,
-              `missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config['xo:base_delta']}`
-            )
-
-            parentPath = parentPath.slice(1) // remove leading slash
-
-            // TODO remove when this has been done before the export
-            await checkVhd(handler, parentPath)
-          }
-
-          transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
+          // don't write it as transferSize += await async function
+          // since i += await asyncFun lead to race condition
+          // as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
+          const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
-
-          if (isDelta) {
-            await chainVhd(handler, parentPath, handler, path)
-          }
-
-          // set the correct UUID in the VHD
-          await Disposable.use(openVhd(handler, path), async vhd => {
-            vhd.footer.uuid = packUuid(vdi.uuid)
-            await vhd.readBlockAllocationTable() // required by writeFooter()
-            await vhd.writeFooter()
-          })
+          transferSize += transferSizeOneDisk
         },
         {
           concurrency: settings.diskPerVmConcurrency,