feat(@vates/nbd-client): export a stream from a remote nbd export

.eslintrc.js

@@ -15,10 +15,9 @@ module.exports = {
 
   overrides: [
     {
-      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js', '**/scripts/**.{,c,m}js'],
+      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
       rules: {
         'n/no-process-exit': 'off',
-        'n/shebang': 'off',
         'no-console': 'off',
       },
     },
@@ -39,12 +38,6 @@ module.exports = {
             version: '>=16',
           },
         ],
-        'n/no-unsupported-features/es-builtins': [
-          'error',
-          {
-            version: '>=16',
-          },
-        ],
         'n/no-unsupported-features/es-syntax': [
           'error',
           {
@@ -53,212 +46,6 @@ module.exports = {
         ],
       },
     },
-    {
-      files: ['@xen-orchestra/{web-core,lite,web}/**/*.{vue,ts}'],
-      parser: 'vue-eslint-parser',
-      parserOptions: {
-        sourceType: 'module',
-        parser: '@typescript-eslint/parser',
-      },
-      plugins: ['import'],
-      extends: [
-        'plugin:import/recommended',
-        'plugin:import/typescript',
-        'plugin:@typescript-eslint/recommended',
-        'plugin:vue/vue3-essential',
-      ],
-      settings: {
-        'import/resolver': {
-          typescript: true,
-          'eslint-import-resolver-custom-alias': {
-            alias: {
-              '@core': '../web-core/lib',
-              '@': './src',
-            },
-            extensions: ['.ts'],
-            packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
-          },
-        },
-      },
-      globals: {
-        XO_LITE_VERSION: true,
-        XO_LITE_GIT_HEAD: true,
-      },
-      rules: {
-        'import/order': [
-          'error',
-          {
-            groups: ['builtin', 'object', 'internal', ['type', 'external']],
-            alphabetize: {
-              order: 'asc',
-              caseInsensitive: true,
-            },
-            pathGroups: [
-              {
-                pattern: '@/**',
-                group: 'internal',
-              },
-              {
-                pattern: '@core/**',
-                group: 'internal',
-              },
-            ],
-          },
-        ],
-        'no-void': ['error', { allowAsStatement: true }],
-        'n/no-missing-import': 'off', // using 'import' plugin instead, to support TS aliases
-        'no-redeclare': 'off', // automatically checked by the TypeScript compiler
-        'no-dupe-class-members': 'off', // automatically checked by the TypeScript compiler
-        'no-undef': 'off', // automatically checked by the TypeScript compiler
-        'no-use-before-define': 'off', // replaced with the rule below
-        '@typescript-eslint/no-use-before-define': ['error', { functions: false }],
-        '@typescript-eslint/no-explicit-any': 'off',
-        'vue/multi-word-component-names': 'off',
-        // Vue 3 - Strongly Recommended
-        'vue/attribute-hyphenation': 'error',
-        'vue/component-definition-name-casing': 'error',
-        'vue/first-attribute-linebreak': 'error',
-        'vue/html-closing-bracket-newline': 'off', // Conflicts with Prettier
-        'vue/html-closing-bracket-spacing': 'error',
-        'vue/html-end-tags': 'error',
-        'vue/html-indent': 'off', // Conflicts with Prettier
-        'vue/html-quotes': 'error',
-        'vue/html-self-closing': ['error', { html: { void: 'always', normal: 'always', component: 'always' } }],
-        'vue/max-attributes-per-line': 'off', // Conflicts with Prettier
-        'vue/multiline-html-element-content-newline': 'error',
-        'vue/mustache-interpolation-spacing': 'error',
-        'vue/no-multi-spaces': 'error',
-        'vue/no-spaces-around-equal-signs-in-attribute': 'error',
-        'vue/no-template-shadow': 'error',
-        'vue/prop-name-casing': 'error',
-        'vue/require-default-prop': 'off', // https://github.com/vuejs/eslint-plugin-vue/issues/2051
-        'vue/require-explicit-emits': 'error',
-        'vue/require-prop-types': 'error',
-        'vue/singleline-html-element-content-newline': 'off',
-        'vue/v-bind-style': 'error',
-        'vue/v-on-event-hyphenation': ['error', 'always', { autofix: true }],
-        'vue/v-on-style': 'error',
-        'vue/v-slot-style': 'error',
-        // Vue 3 - Recommended
-        'vue/attributes-order': 'error',
-        'vue/no-lone-template': 'error',
-        'vue/no-multiple-slot-args': 'error',
-        'vue/no-v-html': 'error',
-        'vue/this-in-template': 'error',
-        // Vue 3 - Uncategorized
-        'vue/block-lang': ['error', { script: { lang: 'ts' }, style: { lang: 'postcss' } }],
-        'vue/block-order': ['error', { order: ['template', 'script', 'style'] }],
-        'vue/block-tag-newline': ['error', { singleline: 'consistent', multiline: 'always', maxEmptyLines: 0 }],
-        'vue/component-api-style': ['error', ['script-setup']],
-        'vue/component-name-in-template-casing': ['error', 'PascalCase', { registeredComponentsOnly: false }],
-        'vue/custom-event-name-casing': ['error', 'camelCase'],
-        'vue/define-emits-declaration': ['error', 'type-literal'],
-        'vue/define-macros-order': [
-          'error',
-          {
-            order: ['defineOptions', 'defineProps', 'defineEmits', 'defineModel', 'defineSlots'],
-            defineExposeLast: true,
-          },
-        ],
-        'vue/define-props-declaration': ['error', 'type-based'],
-        'vue/enforce-style-attribute': ['error', { allow: ['scoped', 'module'] }],
-        'vue/html-button-has-type': 'error',
-        'vue/html-comment-content-newline': ['error', { singleline: 'never', multiline: 'always' }],
-        'vue/html-comment-content-spacing': ['error', 'always'],
-        'vue/no-duplicate-attr-inheritance': 'error',
-        'vue/no-empty-component-block': 'error',
-        'vue/no-multiple-objects-in-class': 'error',
-        'vue/no-ref-object-reactivity-loss': 'error',
-        'vue/no-required-prop-with-default': 'error',
-        'vue/no-static-inline-styles': 'error',
-        'vue/no-template-target-blank': 'error',
-        'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
-        'vue/no-undef-properties': 'error',
-        'vue/no-unused-refs': 'error',
-        'vue/no-use-v-else-with-v-for': 'error',
-        'vue/no-useless-mustaches': 'error',
-        'vue/no-useless-v-bind': 'error',
-        'vue/no-v-text': 'error',
-        'vue/padding-line-between-blocks': 'error',
-        'vue/prefer-define-options': 'error',
-        'vue/prefer-separate-static-class': 'error',
-        'vue/prefer-true-attribute-shorthand': 'error',
-        'vue/require-macro-variable-name': 'error',
-        'vue/valid-define-options': 'error',
-      },
-    },
-    // Specific rules for XO i18n-related files
-    {
-      files: [
-        '@xen-orchestra/lite/**/*.{vue,ts,json}',
-        '@xen-orchestra/web/**/*.{vue,ts,json}',
-        '@xen-orchestra/web-core/**/*.{vue,ts,json}',
-      ],
-      extends: ['plugin:@intlify/vue-i18n/base'],
-      settings: {
-        'vue-i18n': {
-          localeDir: [
-            '@xen-orchestra/lite/src/locales/*.json',
-            '@xen-orchestra/web/src/locales/*.json',
-            '@xen-orchestra/web-core/lib/locales/*.json',
-          ],
-          messageSyntaxVersion: '^9.9.0',
-        },
-      },
-      rules: {
-        strict: 'off',
-        'no-irregular-whitespace': 'off',
-        '@intlify/vue-i18n/no-deprecated-i18n-component': 'error',
-        '@intlify/vue-i18n/no-deprecated-i18n-place-attr': 'error',
-        '@intlify/vue-i18n/no-deprecated-i18n-places-prop': 'error',
-        '@intlify/vue-i18n/no-deprecated-modulo-syntax': 'error',
-        '@intlify/vue-i18n/no-deprecated-tc': 'error',
-        '@intlify/vue-i18n/no-html-messages': 'error',
-        '@intlify/vue-i18n/no-i18n-t-path-prop': 'error',
-        '@intlify/vue-i18n/no-missing-keys': 'error',
-        '@intlify/vue-i18n/no-raw-text': [
-          'error',
-          {
-            ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
-          },
-        ],
-        '@intlify/vue-i18n/no-v-html': 'error',
-        '@intlify/vue-i18n/valid-message-syntax': 'error',
-        '@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
-        '@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa'] }],
-      },
-    },
-    // Specific rules for XO dev-related files
-    {
-      files: [
-        '@xen-orchestra/lite/src/stories/**/*',
-        '@xen-orchestra/lite/src/components/component-story/**/*',
-        '@xen-orchestra/web/src/pages/dev/**/*',
-        '@xen-orchestra/lite/src/views/story/**/*',
-      ],
-      rules: {
-        '@intlify/vue-i18n/no-raw-text': 'off',
-      },
-    },
-    {
-      files: ['@xen-orchestra/{web-core,lite,web}/src/pages/**/*.vue'],
-      parserOptions: {
-        sourceType: 'module',
-      },
-      rules: {
-        'vue/multi-word-component-names': 'off',
-      },
-    },
-    {
-      files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
-      parserOptions: {
-        sourceType: 'module',
-      },
-      rules: {
-        'eslint-comments/disable-enable-pair': 'off',
-        'eslint-comments/no-unlimited-disable': 'off',
-      },
-    },
   ],
 
   parserOptions: {
@@ -281,11 +68,6 @@ module.exports = {
 
     'no-console': ['error', { allow: ['warn', 'error'] }],
 
-    // this rule can prevent race condition bugs like parallel `a += await foo()`
-    //
-    // as it has a lots of false positive, it is only enabled as a warning for now
-    'require-atomic-updates': ['warn', { allowProperties: true }],
-
     strict: 'error',
   },
 }

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,48 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
+assignees: ''
+---
+
+1. ⚠️ **If you don't follow this template, the issue will be closed**.
+2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
+
+Are you using XOA or XO from the sources?
+
+If XOA:
+
+- which release channel? (`stable` vs `latest`)
+- please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
+
+If XO from the sources:
+
+- Provide **your commit number**. If it's older than a week, we won't investigate
+- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
+- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please provide the following information):**
+
+- Node: [e.g. 16.12.1]
+- hypervisor: [e.g. XCP-ng 8.2.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,119 +0,0 @@
-name: Bug Report
-description: Create a report to help us improve
-labels: ['type: bug :bug:', 'status: triaging :triangular_flag_on_post:']
-body:
-  - type: markdown
-    attributes:
-      value: |
-        1. ⚠️ **If you don't follow this template, the issue will be closed**.
-        2. ⚠️ **If your issue can't be easily reproduced, please report it [on the forum first](https://xcp-ng.org/forum/category/12/xen-orchestra)**.
-  - type: markdown
-    attributes:
-      value: '## Are you using XOA or XO from the sources?'
-  - type: dropdown
-    id: xo-origin
-    attributes:
-      label: Are you using XOA or XO from the sources?
-      options:
-        - XOA
-        - XO from the sources
-        - both
-    validations:
-      required: false
-  - type: markdown
-    attributes:
-      value: '### If XOA:'
-  - type: dropdown
-    id: xoa-channel
-    attributes:
-      label: Which release channel?
-      description: please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
-      options:
-        - stable
-        - latest
-        - both
-    validations:
-      required: false
-  - type: markdown
-    attributes:
-      value: '### If XO from the sources:'
-  - type: markdown
-    attributes:
-      value: |
-        - Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
-        - As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
-  - type: input
-    id: xo-sources-commit-number
-    attributes:
-      label: Provide your commit number
-      description: If it's older than a week, we won't investigate
-      placeholder: e.g. 579f0
-    validations:
-      required: false
-  - type: markdown
-    attributes:
-      value: '## Bug description:'
-  - type: textarea
-    id: bug-description
-    attributes:
-      label: Describe the bug
-      description: A clear and concise description of what the bug is
-    validations:
-      required: true
-  - type: textarea
-    id: error-message
-    attributes:
-      label: Error message
-      render: Text
-    validations:
-      required: false
-  - type: textarea
-    id: steps
-    attributes:
-      label: To reproduce
-      description: 'Steps to reproduce the behavior:'
-      value: |
-        1. Go to '...'
-        2. Click on '...'
-        3. Scroll down to '...'
-        4. See error
-    validations:
-      required: false
-  - type: textarea
-    id: expected-behavior
-    attributes:
-      label: Expected behavior
-      description: A clear and concise description of what you expected to happen
-    validations:
-      required: false
-  - type: textarea
-    id: screenshots
-    attributes:
-      label: Screenshots
-      description: If applicable, add screenshots to help explain your problem
-    validations:
-      required: false
-  - type: markdown
-    attributes:
-      value: '## Environment (please provide the following information):'
-  - type: input
-    id: node-version
-    attributes:
-      label: Node
-      placeholder: e.g. 16.12.1
-    validations:
-      required: true
-  - type: input
-    id: hypervisor-version
-    attributes:
-      label: Hypervisor
-      placeholder: e.g. XCP-ng 8.2.0
-    validations:
-      required: true
-  - type: textarea
-    id: additional-context
-    attributes:
-      label: Additional context
-      description: Add any other context about the problem here
-    validations:
-      required: false

.github/workflows/ci.yml

@@ -24,14 +24,8 @@ jobs:
           cache: 'yarn'
       - name: Install project dependencies
         run: yarn
-      - name: Ensure packages are sorted in CHANGELOG.unreleased.md
-        run: ./scripts/gen-deps-list.js --check-order > /dev/null
-      - name: Ensure yarn.lock is up-to-date
-        run: git diff --exit-code yarn.lock
       - name: Build the project
         run: yarn build
-      - name: Unit tests
-        run: yarn test-unit
       - name: Lint tests
         run: yarn test-lint
       - name: Integration tests

.gitignore

@@ -30,12 +30,8 @@ pnpm-debug.log.*
 yarn-error.log
 yarn-error.log.*
 .env
-*.tsbuildinfo
 
 # code coverage
 .nyc_output/
 coverage/
 .turbo/
-
-# https://node-tap.org/dot-tap-folder/
-.tap/

.prettierignore

@@ -1,6 +0,0 @@
-# Ignore Handlebars files because partials are not currently supported
-# https://github.com/prettier/prettier/issues/11834
-**/*.hbs
-
-# Generated files
-/@xen-orchestra/web/typed-router.d.ts

.prettierrc.js

@@ -1,11 +1,8 @@
-'use strict'
-
 module.exports = {
   arrowParens: 'avoid',
   jsxSingleQuote: true,
   semi: false,
   singleQuote: true,
-  trailingComma: 'es5',
 
   // 2020-11-24: Requested by nraynaud and approved by the rest of the team
   //

@vates/async-each/package.json

@@ -33,7 +33,8 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
+    "sinon": "^15.0.1",
+    "tap": "^16.3.0",
     "test": "^3.2.1"
   }
 }

@vates/decorate-with/.USAGE.md

@@ -62,42 +62,6 @@ decorateClass(Foo, {
 })
 ```
 
-### `decorateObject(object, map)`
-
-Decorates an object the same way `decorateClass()` decorates a class:
-
-```js
-import { decorateObject } from '@vates/decorate-with'
-
-const object = {
-  get bar() {
-    // body
-  },
-
-  set bar(value) {
-    // body
-  },
-
-  baz() {
-    // body
-  },
-}
-
-decorateObject(object, {
-  // getter and/or setter
-  bar: {
-    // without arguments
-    get: lodash.memoize,
-
-    // with arguments
-    set: [lodash.debounce, 150],
-  },
-
-  // method (with or without arguments)
-  baz: lodash.curry,
-})
-```
-
 ### `perInstance(fn, ...args)`
 
 Helper to decorate the method by instance instead of for the whole class.

@vates/decorate-with/README.md

@@ -80,42 +80,6 @@ decorateClass(Foo, {
 })
 ```
 
-### `decorateObject(object, map)`
-
-Decorates an object the same way `decorateClass()` decorates a class:
-
-```js
-import { decorateObject } from '@vates/decorate-with'
-
-const object = {
-  get bar() {
-    // body
-  },
-
-  set bar(value) {
-    // body
-  },
-
-  baz() {
-    // body
-  },
-}
-
-decorateObject(object, {
-  // getter and/or setter
-  bar: {
-    // without arguments
-    get: lodash.memoize,
-
-    // with arguments
-    set: [lodash.debounce, 150],
-  },
-
-  // method (with or without arguments)
-  baz: lodash.curry,
-})
-```
-
 ### `perInstance(fn, ...args)`
 
 Helper to decorate the method by instance instead of for the whole class.

@vates/decorate-with/index.js

@@ -14,13 +14,10 @@ function applyDecorator(decorator, value) {
 }
 
 exports.decorateClass = exports.decorateMethodsWith = function decorateClass(klass, map) {
-  return decorateObject(klass.prototype, map)
-}
-
-function decorateObject(object, map) {
+  const { prototype } = klass
   for (const name of Object.keys(map)) {
     const decorator = map[name]
-    const descriptor = getOwnPropertyDescriptor(object, name)
+    const descriptor = getOwnPropertyDescriptor(prototype, name)
     if (typeof decorator === 'function' || Array.isArray(decorator)) {
       descriptor.value = applyDecorator(decorator, descriptor.value)
     } else {
@@ -33,11 +30,10 @@ function decorateObject(object, map) {
       }
     }
 
-    defineProperty(object, name, descriptor)
+    defineProperty(prototype, name, descriptor)
   }
-  return object
+  return klass
 }
-exports.decorateObject = decorateObject
 
 exports.perInstance = function perInstance(fn, decorator, ...args) {
   const map = new WeakMap()

@vates/decorate-with/index.test.js

@@ -13,15 +13,12 @@ describe('decorateWith', () => {
     const expectedFn = Function.prototype
     const newFn = () => {}
 
-    const decorator = decorateWith(
-      function wrapper(fn, ...args) {
-        assert.deepStrictEqual(fn, expectedFn)
-        assert.deepStrictEqual(args, expectedArgs)
+    const decorator = decorateWith(function wrapper(fn, ...args) {
+      assert.deepStrictEqual(fn, expectedFn)
+      assert.deepStrictEqual(args, expectedArgs)
 
-        return newFn
-      },
-      ...expectedArgs
-    )
+      return newFn
+    }, ...expectedArgs)
 
     const descriptor = {
       configurable: true,

@vates/decorate-with/package.json

@@ -20,7 +20,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.1.0",
+  "version": "2.0.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/disposable/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.5",
+  "version": "0.1.4",
   "engines": {
     "node": ">=8.10"
   },
@@ -23,13 +23,13 @@
     "test": "node--test"
   },
   "dependencies": {
-    "@vates/multi-key-map": "^0.2.0",
+    "@vates/multi-key-map": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.6.0",
     "ensure-array": "^1.0.0"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
+    "sinon": "^15.0.1",
     "test": "^3.2.1"
   }
 }

@vates/event-listeners-manager/index.spec.js

@@ -20,9 +20,6 @@ function assertListeners(t, event, listeners) {
 }
 
 t.beforeEach(function (t) {
-  // work around https://github.com/tapjs/tapjs/issues/998
-  t.context = {}
-
   t.context.ee = new EventEmitter()
   t.context.em = new EventListenersManager(t.context.ee)
 })

@vates/event-listeners-manager/package.json

@@ -38,9 +38,9 @@
   "version": "1.0.1",
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "tap --allow-incomplete-coverage"
+    "test": "tap --branches=72"
   },
   "devDependencies": {
-    "tap": "^18.7.0"
+    "tap": "^16.2.0"
   }
 }

@vates/fuse-vhd/.USAGE.md

@@ -1,28 +0,0 @@
-Mount a vhd generated by xen-orchestra to filesystem
-
-### Library
-
-```js
-import { mount } from 'fuse-vhd'
-
-// return a disposable, see promise-toolbox/Disposable
-// unmount automatically when disposable is disposed
-// in case of differencing VHD, it mounts the full chain
-await mount(handler, diskId, mountPoint)
-```
-
-### cli
-
-From the install folder :
-
-```
-cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
-```
-
-After installing the package
-
-```
-xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
-```
-
-remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`

@vates/fuse-vhd/README.md

@@ -1,59 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/fuse-vhd
-
-[![Package Version](https://badgen.net/npm/v/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd) ![License](https://badgen.net/npm/license/@vates/fuse-vhd) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/fuse-vhd)](https://bundlephobia.com/result?p=@vates/fuse-vhd) [![Node compatibility](https://badgen.net/npm/node/@vates/fuse-vhd)](https://npmjs.org/package/@vates/fuse-vhd)
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/fuse-vhd):
-
-```sh
-npm install --save @vates/fuse-vhd
-```
-
-## Usage
-
-Mount a vhd generated by xen-orchestra to filesystem
-
-### Library
-
-```js
-import { mount } from 'fuse-vhd'
-
-// return a disposable, see promise-toolbox/Disposable
-// unmount automatically when disposable is disposed
-// in case of differencing VHD, it mounts the full chain
-await mount(handler, diskId, mountPoint)
-```
-
-### cli
-
-From the install folder :
-
-```
-cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
-```
-
-After installing the package
-
-```
-xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
-```
-
-remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/fuse-vhd/cli.mjs

@@ -1,26 +0,0 @@
-#!/usr/bin/env node
-
-import Disposable from 'promise-toolbox/Disposable'
-import { getSyncedHandler } from '@xen-orchestra/fs'
-
-import { mount } from './index.mjs'
-
-async function* main([remoteUrl, vhdPathInRemote, mountPoint]) {
-  if (mountPoint === undefined) {
-    throw new TypeError('missing arg: cli <remoteUrl> <vhdPathInRemote> <mountPoint>')
-  }
-  const handler = yield getSyncedHandler({ url: remoteUrl })
-  const mounted = await mount(handler, vhdPathInRemote, mountPoint)
-
-  let disposePromise
-  process.on('SIGINT', async () => {
-    // ensure single dispose
-    if (!disposePromise) {
-      disposePromise = mounted.dispose()
-    }
-    await disposePromise
-    process.exit()
-  })
-}
-
-Disposable.wrap(main)(process.argv.slice(2))

@vates/fuse-vhd/index.mjs

@@ -1,11 +1,8 @@
 import LRU from 'lru-cache'
 import Fuse from 'fuse-native'
-import { createLogger } from '@xen-orchestra/log'
 import { VhdSynthetic } from 'vhd-lib'
 import { Disposable, fromCallback } from 'promise-toolbox'
 
-const { warn } = createLogger('vates:fuse-vhd')
-
 // build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
   mtime: st.mtime || new Date(),
@@ -55,16 +52,13 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
     },
     read(path, fd, buf, len, pos, cb) {
       if (path === '/vhd0') {
-        return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
-          warn('read error', { path, len, pos, error })
-          cb(Fuse.EIO)
-        })
+        return vhd.readRawData(pos, len, cache, buf).then(cb)
       }
       throw new Error(`read file ${path} not exists`)
     },
   })
   return new Disposable(
-    () => fromCallback(cb => fuse.unmount(cb)),
-    fromCallback(cb => fuse.mount(cb))
+    () => fromCallback(() => fuse.unmount()),
+    fromCallback(() => fuse.mount())
   )
 })

@vates/fuse-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/fuse-vhd",
-  "version": "2.1.1",
+  "version": "1.0.0",
   "license": "ISC",
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,15 +19,10 @@
   },
   "main": "./index.mjs",
   "dependencies": {
-    "@xen-orchestra/fs": "^4.1.7",
-    "@xen-orchestra/log": "^0.6.0",
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.11.0"
-  },
-  "bin": {
-    "xo-fuse-vhd": "./cli.mjs"
+    "vhd-lib": "^4.5.0"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@vates/json-hash/.USAGE.md

@@ -1,18 +0,0 @@
-The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
-
-The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
-
-The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
-
-```js
-import { jsonHash } from '@vates/json-hash'
-
-console.log(jsonHash('foo'))
-// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
-
-// order of properties is ignored
-console.log(jsonHash({ foo: 0, bar: 1 }))
-// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
-console.log(jsonHash({ bar: 1, foo: 0 }))
-// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
-```

@vates/json-hash/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/json-hash/README.md

@@ -1,51 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/json-hash
-
-[![Package Version](https://badgen.net/npm/v/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash) ![License](https://badgen.net/npm/license/@vates/json-hash) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/json-hash)](https://bundlephobia.com/result?p=@vates/json-hash) [![Node compatibility](https://badgen.net/npm/node/@vates/json-hash)](https://npmjs.org/package/@vates/json-hash)
-
-> Compute a stable hash from a JSON-ifiable value
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/json-hash):
-
-```sh
-npm install --save @vates/json-hash
-```
-
-## Usage
-
-The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
-
-The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
-
-The hash is encoded using [Base64 URL](https://en.wikipedia.org/wiki/Base64#URL_applications) to make it easy to use in URLs and filenames.
-
-```js
-import { jsonHash } from '@vates/json-hash'
-
-console.log(jsonHash('foo'))
-// → "siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs"
-
-// order of properties is ignored
-console.log(jsonHash({ foo: 0, bar: 1 }))
-// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
-console.log(jsonHash({ bar: 1, foo: 0 }))
-// → "JckoRSMIBjNlgEWIXhgpBOuyLQYqABZqvf1ccb3BPg0"
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/json-hash/index.js

@@ -1,38 +0,0 @@
-'use strict'
-
-const { createHash } = require('node:crypto')
-
-function updateJsonHash(value, hash) {
-  if (value !== null && typeof value === 'object') {
-    if (Array.isArray(value)) {
-      hash.update('[')
-      for (const item of value) {
-        updateJsonHash(item, hash)
-
-        // trailing is not a big deal because it does not need to be valid JSON
-        hash.update(',')
-      }
-      hash.update(']')
-    } else {
-      hash.update('{')
-      for (const key of Object.keys(value).sort()) {
-        updateJsonHash(key, hash)
-        hash.update(':')
-        updateJsonHash(value[key], hash)
-
-        // trailing is not a big deal because it does not need to be valid JSON
-        hash.update(',')
-      }
-      hash.update('}')
-    }
-  } else {
-    hash.update(JSON.stringify(value))
-  }
-}
-
-exports.jsonHash = function jsonHash(value) {
-  // this hash does not need to be secure, it just needs to be fast and with low collisions
-  const hash = createHash('sha256')
-  updateJsonHash(value, hash)
-  return hash.digest('base64url')
-}

@vates/json-hash/index.test.js

@@ -1,21 +0,0 @@
-'use strict'
-
-const { jsonHash } = require('@vates/json-hash')
-const assert = require('node:assert/strict')
-const test = require('test')
-
-const cases = [
-  [null, 'dCNOmK_nSY-12vHzasLXiswzlGT5UHA7jAGYkvmCuQs'],
-  [42, 'c0dctApWjo2ooEXO0RATfhWfiQrE2og7axfcZRs6gEk'],
-  ['foo', 'siEyldVkkW-JpqQkVVZ8h8P0gPzXocFeIg8X1xaaeQs'],
-  [[], 'T1PNoYwrqgwDVLtfmj7L5e0Sq02OEbqHPC8RFhICuUU'],
-  [{}, 'RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o'],
-  [{ a: 1, b: 2 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
-  [{ b: 2, a: 1 }, 'LMjVxyj2vtMZ3W1cZ2cJrQij5zn7FBOlPiyBB38OmFE'],
-]
-
-for (const [value, hash] of cases) {
-  test(JSON.stringify(value), function () {
-    assert.equal(jsonHash(value), hash)
-  })
-}

@vates/json-hash/package.json

@@ -1,33 +0,0 @@
-{
-  "private": false,
-  "name": "@vates/json-hash",
-  "description": "Compute a stable hash from a JSON-ifiable value",
-  "keywords": [
-    "hash",
-    "json",
-    "stable"
-  ],
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/json-hash",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/json-hash",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "0.2.0",
-  "engines": {
-    "node": ">=14.18.0"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
-  "scripts": {
-    "postversion": "npm publish --access public",
-    "test": "node--test"
-  }
-}

@vates/multi-key-map/.USAGE.md

@@ -17,14 +17,4 @@ map.get(['foo', 'bar']) // 2
 map.get(['bar', 'foo']) // 3
 map.get([OBJ]) // 4
 map.get([{}]) // undefined
-
-map.delete([])
-
-for (const [key, value] of map.entries() {
-  console.log(key, value)
-}
-
-for (const  value of map.values()) {
-  console.log(value)
-}
 ```

@vates/multi-key-map/README.md

@@ -35,16 +35,6 @@ map.get(['foo', 'bar']) // 2
 map.get(['bar', 'foo']) // 3
 map.get([OBJ]) // 4
 map.get([{}]) // undefined
-
-map.delete([])
-
-for (const [key, value] of map.entries() {
-  console.log(key, value)
-}
-
-for (const  value of map.values()) {
-  console.log(value)
-}
 ```
 
 ## Contributions

@vates/multi-key-map/index.js

@@ -36,31 +36,14 @@ function del(node, i, keys) {
   return node
 }
 
-function* entries(node, key) {
-  if (node !== undefined) {
-    if (node instanceof Node) {
-      const { value } = node
-      if (value !== undefined) {
-        yield [key, node.value]
-      }
-
-      for (const [childKey, child] of node.children.entries()) {
-        yield* entries(child, key.concat(childKey))
-      }
-    } else {
-      yield [key, node]
-    }
-  }
-}
-
 function get(node, i, keys) {
   return i === keys.length
     ? node instanceof Node
       ? node.value
       : node
     : node instanceof Node
-      ? get(node.children.get(keys[i]), i + 1, keys)
-      : undefined
+    ? get(node.children.get(keys[i]), i + 1, keys)
+    : undefined
 }
 
 function set(node, i, keys, value) {
@@ -86,22 +69,6 @@ function set(node, i, keys, value) {
   return node
 }
 
-function* values(node) {
-  if (node !== undefined) {
-    if (node instanceof Node) {
-      const { value } = node
-      if (value !== undefined) {
-        yield node.value
-      }
-      for (const child of node.children.values()) {
-        yield* values(child)
-      }
-    } else {
-      yield node
-    }
-  }
-}
-
 exports.MultiKeyMap = class MultiKeyMap {
   constructor() {
     // each node is either a value or a Node if it contains children
@@ -112,10 +79,6 @@ exports.MultiKeyMap = class MultiKeyMap {
     this._root = del(this._root, 0, keys)
   }
 
-  entries() {
-    return entries(this._root, [])
-  }
-
   get(keys) {
     return get(this._root, 0, keys)
   }
@@ -123,8 +86,4 @@ exports.MultiKeyMap = class MultiKeyMap {
   set(keys, value) {
     this._root = set(this._root, 0, keys, value)
   }
-
-  values() {
-    return values(this._root)
-  }
 }

@vates/multi-key-map/index.test.js

@@ -19,7 +19,7 @@ describe('MultiKeyMap', () => {
       // reverse composite key
       ['bar', 'foo'],
     ]
-    const values = keys.map(() => Math.random())
+    const values = keys.map(() => ({}))
 
     // set all values first to make sure they are all stored and not only the
     // last one
@@ -27,12 +27,6 @@ describe('MultiKeyMap', () => {
       map.set(key, values[i])
     })
 
-    assert.deepEqual(
-      Array.from(map.entries()),
-      keys.map((key, i) => [key, values[i]])
-    )
-    assert.deepEqual(Array.from(map.values()), values)
-
     keys.forEach((key, i) => {
       // copy the key to make sure the array itself is not the key
       assert.strictEqual(map.get(key.slice()), values[i])

@vates/multi-key-map/package.json

@@ -18,7 +18,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.2.0",
+  "version": "0.1.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/nbd-client/index.mjs

@@ -1,9 +1,10 @@
 import assert from 'node:assert'
 import { Socket } from 'node:net'
 import { connect } from 'node:tls'
-import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
+import { fromCallback, pRetry, pDelay, pTimeout } from 'promise-toolbox'
 import { readChunkStrict } from '@vates/read-chunk'
 import { createLogger } from '@xen-orchestra/log'
+
 import {
   INIT_PASSWD,
   NBD_CMD_READ,
@@ -20,6 +21,8 @@ import {
   OPTS_MAGIC,
   NBD_CMD_DISC,
 } from './constants.mjs'
+import { Readable } from 'node:stream'
+
 const { warn } = createLogger('vates:nbd-client')
 
 // documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -34,10 +37,10 @@ export default class NbdClient {
   #exportSize
 
   #waitBeforeReconnect
+  #readAhead
   #readBlockRetries
   #reconnectRetry
   #connectTimeout
-  #messageTimeout
 
   // AFAIK, there is no guaranty the server answers in the same order as the queries
   // so we handle a backlog of command waiting for response and handle concurrency manually
@@ -50,23 +53,17 @@ export default class NbdClient {
   #reconnectingPromise
   constructor(
     { address, port = NBD_DEFAULT_PORT, exportname, cert },
-    {
-      connectTimeout = 6e4,
-      messageTimeout = 6e4,
-      waitBeforeReconnect = 1e3,
-      readBlockRetries = 5,
-      reconnectRetry = 5,
-    } = {}
+    { connectTimeout = 6e4, waitBeforeReconnect = 1e3, readAhead = 10, readBlockRetries = 5, reconnectRetry = 5 } = {}
   ) {
     this.#serverAddress = address
     this.#serverPort = port
     this.#exportName = exportname
     this.#serverCert = cert
     this.#waitBeforeReconnect = waitBeforeReconnect
+    this.#readAhead = readAhead
     this.#readBlockRetries = readBlockRetries
     this.#reconnectRetry = reconnectRetry
     this.#connectTimeout = connectTimeout
-    this.#messageTimeout = messageTimeout
   }
 
   get exportSize() {
@@ -119,27 +116,13 @@ export default class NbdClient {
     if (!this.#connected) {
       return
     }
-    this.#connected = false
-    const socket = this.#serverSocket
-
-    const queryId = this.#nextCommandQueryId
-    this.#nextCommandQueryId++
 
     const buffer = Buffer.alloc(28)
     buffer.writeInt32BE(NBD_REQUEST_MAGIC, 0) // it is a nbd request
     buffer.writeInt16BE(0, 4) // no command flags for a disconnect
     buffer.writeInt16BE(NBD_CMD_DISC, 6) // we want to disconnect from nbd server
-    buffer.writeBigUInt64BE(queryId, 8)
-    buffer.writeBigUInt64BE(0n, 16)
-    buffer.writeInt32BE(0, 24)
-    const promise = pFromCallback(cb => {
-      socket.end(buffer, 'utf8', cb)
-    })
-    try {
-      await pTimeout.call(promise, this.#messageTimeout)
-    } catch (error) {
-      socket.destroy()
-    }
+    await this.#write(buffer)
+    await this.#serverSocket.destroy()
     this.#serverSocket = undefined
     this.#connected = false
   }
@@ -213,13 +196,11 @@ export default class NbdClient {
   }
 
   #read(length) {
-    const promise = readChunkStrict(this.#serverSocket, length)
-    return pTimeout.call(promise, this.#messageTimeout)
+    return readChunkStrict(this.#serverSocket, length)
   }
 
   #write(buffer) {
-    const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
-    return pTimeout.call(promise, this.#messageTimeout)
+    return fromCallback.call(this.#serverSocket, 'write', buffer)
   }
 
   async #readInt32() {
@@ -252,20 +233,20 @@ export default class NbdClient {
     }
     try {
       this.#waitingForResponse = true
-      const buffer = await this.#read(16)
+      const buffer = await this.#read(8)
       const magic = buffer.readInt32BE(0)
 
       if (magic !== NBD_REPLY_MAGIC) {
         throw new Error(`magic number for block answer is wrong : ${magic} ${NBD_REPLY_MAGIC}`)
       }
 
-      const error = buffer.readInt32BE(4)
+      const error = buffer.readInt32BE(1)
       if (error !== 0) {
         // @todo use error code from constants.mjs
         throw new Error(`GOT ERROR CODE  : ${error}`)
       }
 
-      const blockQueryId = buffer.readBigUInt64BE(8)
+      const blockQueryId = buffer.readBigUInt64BE(4)
       const query = this.#commandQueryBacklog.get(blockQueryId)
       if (!query) {
         throw new Error(` no query associated with id ${blockQueryId}`)
@@ -286,7 +267,7 @@ export default class NbdClient {
     }
   }
 
-  async #readBlock(index, size) {
+  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
     // we don't want to add anything in backlog while reconnecting
     if (this.#reconnectingPromise) {
       await this.#reconnectingPromise
@@ -334,13 +315,57 @@ export default class NbdClient {
     })
   }
 
-  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
-    return pRetry(() => this.#readBlock(index, size), {
-      tries: this.#readBlockRetries,
-      onRetry: async err => {
-        warn('will retry reading block ', index, err)
-        await this.reconnect()
-      },
-    })
+  async *readBlocks(indexGenerator = 2 * 1024 * 1024) {
+    // default : read all blocks
+    if (typeof indexGenerator === 'number') {
+      const exportSize = Number(this.#exportSize)
+      const chunkSize = indexGenerator
+
+      indexGenerator = function* () {
+        const nbBlocks = Math.ceil(exportSize / chunkSize)
+        for (let index = 0; index < nbBlocks; index++) {
+          yield { index, size: chunkSize }
+        }
+      }
+    }
+    const readAhead = []
+    const readAheadMaxLength = this.#readAhead
+    const makeReadBlockPromise = (index, size) => {
+      const promise = pRetry(() => this.readBlock(index, size), {
+        tries: this.#readBlockRetries,
+        onRetry: async err => {
+          warn('will retry reading block ', index, err)
+          await this.reconnect()
+        },
+      })
+      // error is handled during unshift
+      promise.catch(() => {})
+      return promise
+    }
+
+    // read all blocks, but try to keep readAheadMaxLength promise waiting ahead
+    for (const { index, size } of indexGenerator()) {
+      // stack readAheadMaxLength promises before starting to handle the results
+      if (readAhead.length === readAheadMaxLength) {
+        // any error will stop reading blocks
+        yield readAhead.shift()
+      }
+
+      readAhead.push(makeReadBlockPromise(index, size))
+    }
+    while (readAhead.length > 0) {
+      yield readAhead.shift()
+    }
+  }
+
+  stream(chunk_size) {
+    async function* iterator() {
+      for await (const chunk of this.readBlocks(chunk_size)) {
+        yield chunk
+      }
+    }
+    // create a readable stream instead of returning the iterator
+    // since iterators don't like unshift and partial reading
+    return Readable.from(iterator())
   }
 }

@vates/nbd-client/multi.mjs

@@ -1,116 +0,0 @@
-import { asyncEach } from '@vates/async-each'
-import { NBD_DEFAULT_BLOCK_SIZE } from './constants.mjs'
-import NbdClient from './index.mjs'
-import { createLogger } from '@xen-orchestra/log'
-
-const { warn } = createLogger('vates:nbd-client:multi')
-export default class MultiNbdClient {
-  #clients = []
-  #nbdConcurrency
-  #options
-  #readAhead
-  #settings
-
-  get exportSize() {
-    return this.#clients[0].exportSize
-  }
-
-  constructor(settings, { nbdConcurrency = 8, readAhead = 16, ...options } = {}) {
-    this.#readAhead = readAhead
-    this.#options = options
-    this.#nbdConcurrency = nbdConcurrency
-    if (!Array.isArray(settings)) {
-      settings = [settings]
-    }
-    this.#settings = settings
-  }
-
-  /**
-   *
-   * open nbdConcurrency connections to NBD servers
-   * it must obtain at least one connection to succeed
-   * it tries to spread connections on multiple host
-   */
-  async connect() {
-    const candidates = [...this.#settings]
-
-    const promises = []
-    const baseOptions = this.#options
-    const  _connect = async () => {
-      if (candidates.length === 0) {
-        return
-      }
-      // a little bit of randomization to spread the load
-      const nbdInfo = candidates[Math.floor(Math.random() * candidates.length)]
-      const client = new NbdClient(nbdInfo, {
-        ...baseOptions,
-        readAhead: Math.ceil(this.#readAhead / this.#nbdConcurrency),
-      })
-      try {
-        await client.connect()
-        this.#clients.push(client)
-      } catch (err) {
-        client.disconnect().catch(() => {})
-        // do not hammer unreachable hosts, once failed, remove from the list
-        const candidateIndex = candidates.findIndex(({ address }) => address === nbdInfo.address)
-        if (candidateIndex >= 0) {
-          // this candidate may have already been deleted by another parallel promise
-          candidates.splice(candidateIndex, 1)
-        }
-
-        warn(`can't connect to one nbd client`, { err })
-        // retry with another candidate (if available)
-        return _connect()
-      }
-    }
-    for (let i = 0; i < this.#nbdConcurrency; i++) {
-      promises.push(_connect())
-    }
-    await Promise.all(promises)
-
-    if (this.#clients.length === 0) {
-      throw new Error(`Fail to connect to any Nbd client`)
-    }
-    if (this.#clients.length < this.#nbdConcurrency) {
-      warn(
-        `incomplete connection by multi Nbd, only ${this.#clients.length} over ${this.#nbdConcurrency} expected clients`
-      )
-    }
-  }
-
-  async disconnect() {
-    await asyncEach(this.#clients, client => client.disconnect(), {
-      stopOnError: false,
-    })
-  }
-
-  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
-    const clientId = index % this.#clients.length
-    return this.#clients[clientId].readBlock(index, size)
-  }
-
-  async *readBlocks(indexGenerator) {
-    // default : read all blocks
-    const readAhead = []
-    const makeReadBlockPromise = (index, size) => {
-      const promise = this.readBlock(index, size)
-      // error is handled during unshift
-      promise.catch(() => {})
-      return promise
-    }
-
-    // read all blocks, but try to keep readAheadMaxLength promise waiting ahead
-    for (const { index, size } of indexGenerator()) {
-      // stack readAheadMaxLength promises before starting to handle the results
-      if (readAhead.length === this.#readAhead) {
-        // any error will stop reading blocks
-        yield readAhead.shift()
-      }
-
-      readAhead.push(makeReadBlockPromise(index, size))
-    }
-    while (readAhead.length > 0) {
-      yield readAhead.shift()
-    }
-  }
-}

@vates/nbd-client/package.json

@@ -13,25 +13,25 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "3.1.0",
+  "version": "1.2.1",
   "engines": {
     "node": ">=14.0"
   },
   "main": "./index.mjs",
   "dependencies": {
     "@vates/async-each": "^1.0.0",
-    "@vates/read-chunk": "^1.2.0",
+    "@vates/read-chunk": "^1.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.6.0",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^4.3.1"
+    "xen-api": "^1.3.3"
   },
   "devDependencies": {
-    "tap": "^18.7.0",
+    "tap": "^16.3.0",
     "tmp": "^0.2.1"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "tap --allow-incomplete-coverage"
+    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
   }
 }

@vates/nbd-client/tests/nbdclient.integ.mjs

@@ -1,3 +1,4 @@
+import NbdClient from '../index.mjs'
 import { spawn, exec } from 'node:child_process'
 import fs from 'node:fs/promises'
 import { test } from 'tap'
@@ -6,10 +7,8 @@ import { pFromCallback } from 'promise-toolbox'
 import { Socket } from 'node:net'
 import { NBD_DEFAULT_PORT } from '../constants.mjs'
 import assert from 'node:assert'
-import MultiNbdClient from '../multi.mjs'
 
-const CHUNK_SIZE = 1024 * 1024 // non default size
-const FILE_SIZE = 1024 * 1024 * 9.5 // non aligned file size
+const FILE_SIZE = 10 * 1024 * 1024
 
 async function createTempFile(size) {
   const tmpPath = await pFromCallback(cb => tmp.file(cb))
@@ -78,14 +77,15 @@ async function killNbdKit() {
   )
 }
 
-test('it works secured network', async tap => {
+test('it works with unsecured network', async tap => {
   const path = await createTempFile(FILE_SIZE)
 
   let nbdServer = await spawnNbdKit(path)
-  const connectionSettings = {
-    address: '127.0.0.1',
-    exportname: 'MY_SECRET_EXPORT',
-    cert: `-----BEGIN CERTIFICATE-----
+  const client = new NbdClient(
+    {
+      address: '127.0.0.1',
+      exportname: 'MY_SECRET_EXPORT',
+      cert: `-----BEGIN CERTIFICATE-----
 MIIDazCCAlOgAwIBAgIUeHpQ0IeD6BmP2zgsv3LV3J4BI/EwDQYJKoZIhvcNAQEL
 BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
 GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA1MTcxMzU1MzBaFw0yNDA1
@@ -107,21 +107,15 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
 57ABhfYpMlZ4aHjuN1bL
 -----END CERTIFICATE-----
 `,
-  }
-  const invalid = {
-    address: '500.500.500.500',
-    port: 0,
-    exportname: 'nop',
-  }
-  // it should work even with some broken servers
-  const nbdInfos = [connectionSettings, connectionSettings, invalid]
-  const client = new MultiNbdClient(nbdInfos, {
-    nbdConcurrency: 4,
-    readAhead: 2,
-  })
+    },
+    {
+      readAhead: 2,
+    }
+  )
 
   await client.connect()
   tap.equal(client.exportSize, BigInt(FILE_SIZE))
+  const CHUNK_SIZE = 1024 * 1024 // non default size
   const indexes = []
   for (let i = 0; i < FILE_SIZE / CHUNK_SIZE; i++) {
     indexes.push(i)
@@ -133,9 +127,9 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
   })
   let i = 0
   for await (const block of nbdIterator) {
-    let blockOk = block.length === Math.min(CHUNK_SIZE, FILE_SIZE - CHUNK_SIZE * i)
+    let blockOk = true
     let firstFail
-    for (let j = 0; j < block.length; j += 4) {
+    for (let j = 0; j < CHUNK_SIZE; j += 4) {
       const wanted = i * CHUNK_SIZE + j
       const found = block.readUInt32BE(j)
       blockOk = blockOk && found === wanted
@@ -143,7 +137,7 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
         firstFail = j
       }
     }
-    tap.ok(blockOk, `check block ${i} content ${block.length}`)
+    tap.ok(blockOk, `check block ${i} content`)
     i++
 
     // flaky server is flaky
@@ -153,6 +147,17 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
       nbdServer = await spawnNbdKit(path)
     }
   }
+
+  // we can reuse the conneciton to read other blocks
+  // default iterator
+  const nbdIteratorWithDefaultBlockIterator = client.readBlocks()
+  let nb = 0
+  for await (const block of nbdIteratorWithDefaultBlockIterator) {
+    nb++
+    tap.equal(block.length, 2 * 1024 * 1024)
+  }
+
+  tap.equal(nb, 5)
   assert.rejects(() => client.readBlock(100, CHUNK_SIZE))
 
   await client.disconnect()
@@ -161,18 +166,3 @@ CYu1Xn/FVPx1HoRgWc7E8wFhDcA/P3SJtfIQWHB9FzSaBflKGR4t8WCE2eE8+cTB
   nbdServer.kill()
   await fs.unlink(path)
 })
-
-test('fails if server does not answer', async () => {
-  const client = new MultiNbdClient(
-    {
-      address: '500.500.500.500',
-      port: 0,
-      exportname: 'nop',
-    },
-    {
-      nbdConcurrency: 1,
-      readAhead: 2,
-    }
-  )
-  await assert.rejects(client.connect())
-})

@vates/node-vsphere-soap/lib/client.mjs

@@ -17,20 +17,7 @@ import https from 'node:https'
 import util from 'util'
 import soap from 'soap'
 import Cookie from 'soap-cookie' // required for session persistence
-import { createLogger } from '@xen-orchestra/log'
 
-const { warn } = createLogger('xo:node-vsphere-soap:client')
-class VmwareError extends Error {
-  constructor(rawError) {
-    super(rawError.body)
-    const matches = rawError.body?.match(/<faultstring[^>]*>(.*)<\/faultstring>/im)
-    this.message = matches?.[1] ?? rawError.message
-    this.name = 'VmwareError'
-    this.stack = rawError.stack
-    // not putting the cause since the error failed to be stringified correctly down the road
-    warn(rawError)
-  }
-}
 // Client class
 // inherits from EventEmitter
 // possible events: connect, error, ready
@@ -44,20 +31,14 @@ export function Client(vCenterHostname, username, password, sslVerify) {
   EventEmitter.call(this)
 
   // sslVerify argument handling
-  // don't use any proxy to connect to the esxi
   if (sslVerify) {
-    this.clientopts = {
-      request: axios.create({
-        proxy: false,
-      }),
-    }
+    this.clientopts = {}
   } else {
     this.clientopts = {
       request: axios.create({
         httpsAgent: new https.Agent({
           rejectUnauthorized: false,
         }),
-        proxy: false,
       }),
     }
   }
@@ -153,9 +134,8 @@ Client.prototype._connect = function () {
     self.clientopts,
     function (err, client) {
       if (err) {
-        const vErr = new VmwareError(err)
-        self.emit('error', vErr)
-        throw vErr
+        self.emit('error', err)
+        throw err
       }
 
       self.client = client // save client for later use
@@ -164,7 +144,6 @@ Client.prototype._connect = function () {
         .runCommand('RetrieveServiceContent', { _this: 'ServiceInstance' })
         .once('result', function (result, raw, soapHeader) {
           if (!result.returnval) {
-            // this is a domain/server error return it as is
             self.status = 'disconnected'
             self.emit('error', raw)
             return
@@ -223,7 +202,7 @@ Client.prototype._close = function () {
 function _soapErrorHandler(self, emitter, command, args, err) {
   err = err || { body: 'general error' }
 
-  if (err.body?.match(/session is not authenticated/)) {
+  if (err.body.match(/session is not authenticated/)) {
     self.status = 'disconnected'
     process.removeAllListeners('beforeExit')
 
@@ -235,19 +214,16 @@ function _soapErrorHandler(self, emitter, command, args, err) {
           emitter.emit('result', result, raw, soapHeader)
         })
         .once('error', function (err) {
-          const vErr = new VmwareError(err)
-          emitter.emit('error', vErr)
-          throw vErr
+          emitter.emit('error', err.body)
+          throw err
         })
     } else {
-      const vErr = new VmwareError(err)
-      emitter.emit('error', vErr)
-      throw vErr
+      emitter.emit('error', err.body)
+      throw err
     }
   } else {
-    const vErr = new VmwareError(err)
-    emitter.emit('error', vErr)
-    throw vErr
+    emitter.emit('error', err.body)
+    throw err
   }
 }
 

@vates/node-vsphere-soap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/node-vsphere-soap",
-  "version": "2.1.0",
+  "version": "1.0.0",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
   "main": "lib/client.mjs",
   "author": "reedog117",
@@ -11,7 +11,6 @@
   },
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
-    "@xen-orchestra/log": "^0.6.0",
     "axios": "^1.4.0",
     "soap": "^1.0.0",
     "soap-cookie": "^0.10.1"

@vates/obfuscate/.USAGE.md

@@ -1,46 +0,0 @@
-### `replace(value, replacement)`
-
-> Recursively replace sensitive entries with a replacement value.
-
-```js
-import { replace } from '@vates/obfuscate'
-
-replace({ username: 'foo', password: 'bar' }, 'obfuscated')
-// → { username: 'foo', password: 'obfuscated'}
-```
-
-### `obfuscate(value)`
-
-> Recursively replace sensitive entries with a special tag.
-
-```js
-import { obfuscate } from '@vates/obfuscate'
-
-obfuscate({ username: 'foo', password: 'bar' })
-// → {
-//   username: 'foo',
-//   password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
-// }
-```
-
-### `merge(newValue, oldValue)`
-
-> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
->
-> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
-
-```js
-import { obfuscate, merge } from '@vates/obfuscate'
-
-const original = { username: 'foo', password: 'bar', token: 'baz' }
-
-// compute an obfuscated value to send to a client
-const obfuscated = obfuscate(original)
-
-// let the client update any entries
-obfuscated.token = 'qux'
-
-// merge the updated value with the original one
-merge(obfuscated, original)
-// → { username: 'foo', password: 'bar', token: 'qux' }
-```

@vates/obfuscate/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/obfuscate/README.md

@@ -1,77 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/obfuscate
-
-[![Package Version](https://badgen.net/npm/v/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate) ![License](https://badgen.net/npm/license/@vates/obfuscate) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/obfuscate)](https://bundlephobia.com/result?p=@vates/obfuscate) [![Node compatibility](https://badgen.net/npm/node/@vates/obfuscate)](https://npmjs.org/package/@vates/obfuscate)
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/obfuscate):
-
-```sh
-npm install --save @vates/obfuscate
-```
-
-## Usage
-
-### `replace(value, replacement)`
-
-> Recursively replace sensitive entries with a replacement value.
-
-```js
-import { replace } from '@vates/obfuscate'
-
-replace({ username: 'foo', password: 'bar' }, 'obfuscated')
-// → { username: 'foo', password: 'obfuscated'}
-```
-
-### `obfuscate(value)`
-
-> Recursively replace sensitive entries with a special tag.
-
-```js
-import { obfuscate } from '@vates/obfuscate'
-
-obfuscate({ username: 'foo', password: 'bar' })
-// → {
-//   username: 'foo',
-//   password: 'obfuscated-q3oi6d9X8uenGvdLnHk2',
-// }
-```
-
-### `merge(newValue, oldValue)`
-
-> Replace entries obfuscated by `obfuscate()` in the new value by entries from the old one.
->
-> This is especially useful to give an obfuscated value to a client, let them modify it, and then merge modified and unmodified values.
-
-```js
-import { obfuscate, merge } from '@vates/obfuscate'
-
-const original = { username: 'foo', password: 'bar', token: 'baz' }
-
-// compute an obfuscated value to send to a client
-const obfuscated = obfuscate(original)
-
-// let the client update any entries
-obfuscated.token = 'qux'
-
-// merge the updated value with the original one
-merge(obfuscated, original)
-// → { username: 'foo', password: 'bar', token: 'qux' }
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/obfuscate/package.json

@@ -1,23 +0,0 @@
-{
-  "private": false,
-  "name": "@vates/obfuscate",
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/obfuscate",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/obfuscate",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "0.1.0",
-  "engines": {
-    "node": ">=8.10"
-  },
-  "scripts": {
-    "postversion": "npm publish --access public"
-  }
-}

@vates/otp/index.mjs

@@ -9,15 +9,6 @@ function assert(name, value) {
   }
 }
 
-const WHITESPACES = /\s+/g
-function cleanToken(otp) {
-  // remove whitespaces
-  //
-  // some OTP clients add whitespaces for better readability (e.g. `XXX XXX`)
-  // and users might be tempted to write them
-  return otp && otp.replace(WHITESPACES, '')
-}
-
 // https://github.com/google/google-authenticator/wiki/Key-Uri-Format
 function generateUri(protocol, label, params) {
   assert('label', typeof label === 'string')
@@ -73,7 +64,7 @@ export function generateHotpUri({ counter, digits, issuer, label, secret }) {
 }
 
 export async function verifyHotp(token, opts) {
-  return cleanToken(token) === (await generateHotp(opts))
+  return token === (await generateHotp(opts))
 }
 
 function totpCounter(period = 30, timestamp = Math.floor(Date.now() / 1e3)) {
@@ -94,7 +85,6 @@ export async function verifyTotp(token, { period, timestamp, window = 1, ...opts
   const counter = totpCounter(period, timestamp)
   const end = counter + window
   opts.counter = counter - window
-  token = cleanToken(token)
   while (opts.counter <= end) {
     if (token === (await generateHotp(opts))) {
       return true

@vates/otp/index.spec.mjs

@@ -1,5 +1,5 @@
 import { strict as assert } from 'node:assert'
-import test from 'test'
+import { describe, it } from 'tap/mocha'
 
 import {
   generateHotp,
@@ -11,8 +11,6 @@ import {
   verifyTotp,
 } from './index.mjs'
 
-const { describe, it } = test
-
 describe('generateSecret', function () {
   it('generates a string of 32 chars', async function () {
     const secret = generateSecret()

@vates/otp/package.json

@@ -22,7 +22,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.1.0",
+  "version": "1.0.0",
   "engines": {
     "node": ">=15"
   },
@@ -31,9 +31,9 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "tap"
   },
   "devDependencies": {
-    "test": "^3.3.0"
+    "tap": "^16.3.0"
   }
 }

@vates/parse-duration/index.js

@@ -6,11 +6,9 @@ exports.parseDuration = value => {
   if (typeof value === 'number') {
     return value
   }
-  if (typeof value === 'string' && value !== '') {
-    const duration = ms(value)
-    if (duration !== undefined) {
-      return duration
-    }
+  const duration = ms(value)
+  if (duration === undefined) {
+    throw new TypeError(`not a valid duration: ${value}`)
   }
-  throw new TypeError(`not a valid duration: ${value}`)
+  return duration
 }

@vates/parse-duration/index.test.mjs

@@ -1,22 +0,0 @@
-import { describe, it } from 'node:test'
-import { parseDuration } from '@vates/parse-duration'
-import assert from 'node:assert/strict'
-
-describe('parseDuration()', () => {
-  it('should parse string', () => {
-    const input = '2 days'
-    const expected = 172800000
-    assert.strictEqual(parseDuration(input), expected)
-  })
-
-  it('should return its input if already a number', () => {
-    const input = 172800000
-    assert.strictEqual(parseDuration(input), input)
-  })
-
-  for (const input of [undefined, '', 'invalid duration']) {
-    it('should throw an error for ' + input, () => {
-      assert.throws(() => parseDuration(input), { message: `not a valid duration: ${input}` })
-    })
-  }
-})

@vates/parse-duration/package.json

@@ -27,7 +27,6 @@
     "ms": "^2.1.2"
   },
   "scripts": {
-    "postversion": "npm publish --access public",
-    "test": "node --test"
+    "postversion": "npm publish --access public"
   }
 }

@vates/predicates/index.spec.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { describe, it } = require('test')
+const { describe, it } = require('tap').mocha
 
 const { every, not, some } = require('./')
 

@vates/predicates/package.json

@@ -32,9 +32,9 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "tap"
   },
   "devDependencies": {
-    "test": "^3.3.0"
+    "tap": "^16.0.1"
   }
 }

@vates/read-chunk/index.js

@@ -22,41 +22,41 @@ const readChunk = (stream, size) =>
   stream.errored != null
     ? Promise.reject(stream.errored)
     : stream.closed || stream.readableEnded
-      ? Promise.resolve(null)
-      : new Promise((resolve, reject) => {
-          if (size !== undefined) {
-            assert(size > 0)
+    ? Promise.resolve(null)
+    : new Promise((resolve, reject) => {
+        if (size !== undefined) {
+          assert(size > 0)
 
-            // per Node documentation:
-            // > The size argument must be less than or equal to 1 GiB.
-            assert(size < 1073741824)
-          }
+          // per Node documentation:
+          // > The size argument must be less than or equal to 1 GiB.
+          assert(size < 1073741824)
+        }
 
-          function onEnd() {
-            resolve(null)
+        function onEnd() {
+          resolve(null)
+          removeListeners()
+        }
+        function onError(error) {
+          reject(error)
+          removeListeners()
+        }
+        function onReadable() {
+          const data = stream.read(size)
+          if (data !== null) {
+            resolve(data)
             removeListeners()
           }
-          function onError(error) {
-            reject(error)
-            removeListeners()
-          }
-          function onReadable() {
-            const data = stream.read(size)
-            if (data !== null) {
-              resolve(data)
-              removeListeners()
-            }
-          }
-          function removeListeners() {
-            stream.removeListener('end', onEnd)
-            stream.removeListener('error', onError)
-            stream.removeListener('readable', onReadable)
-          }
-          stream.on('end', onEnd)
-          stream.on('error', onError)
-          stream.on('readable', onReadable)
-          onReadable()
-        })
+        }
+        function removeListeners() {
+          stream.removeListener('end', onEnd)
+          stream.removeListener('error', onError)
+          stream.removeListener('readable', onReadable)
+        }
+        stream.on('end', onEnd)
+        stream.on('error', onError)
+        stream.on('readable', onReadable)
+        onReadable()
+      })
 exports.readChunk = readChunk
 
 /**
@@ -111,42 +111,42 @@ async function skip(stream, size) {
   return stream.errored != null
     ? Promise.reject(stream.errored)
     : size === 0 || stream.closed || stream.readableEnded
-      ? Promise.resolve(0)
-      : new Promise((resolve, reject) => {
-          let left = size
-          function onEnd() {
-            resolve(size - left)
-            removeListeners()
-          }
-          function onError(error) {
-            reject(error)
-            removeListeners()
-          }
-          function onReadable() {
-            const data = stream.read()
-            left -= data === null ? 0 : data.length
-            if (left > 0) {
-              // continue to read
-            } else {
-              // if more than wanted has been read, push back the rest
-              if (left < 0) {
-                stream.unshift(data.slice(left))
-              }
-
-              resolve(size)
-              removeListeners()
+    ? Promise.resolve(0)
+    : new Promise((resolve, reject) => {
+        let left = size
+        function onEnd() {
+          resolve(size - left)
+          removeListeners()
+        }
+        function onError(error) {
+          reject(error)
+          removeListeners()
+        }
+        function onReadable() {
+          const data = stream.read()
+          left -= data === null ? 0 : data.length
+          if (left > 0) {
+            // continue to read
+          } else {
+            // if more than wanted has been read, push back the rest
+            if (left < 0) {
+              stream.unshift(data.slice(left))
             }
+
+            resolve(size)
+            removeListeners()
           }
-          function removeListeners() {
-            stream.removeListener('end', onEnd)
-            stream.removeListener('error', onError)
-            stream.removeListener('readable', onReadable)
-          }
-          stream.on('end', onEnd)
-          stream.on('error', onError)
-          stream.on('readable', onReadable)
-          onReadable()
-        })
+        }
+        function removeListeners() {
+          stream.removeListener('end', onEnd)
+          stream.removeListener('error', onError)
+          stream.removeListener('readable', onReadable)
+        }
+        stream.on('end', onEnd)
+        stream.on('error', onError)
+        stream.on('readable', onReadable)
+        onReadable()
+      })
 }
 exports.skip = skip
 

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "1.2.0",
+  "version": "1.1.1",
   "engines": {
     "node": ">=8.10"
   },

@vates/stream-reader/package.json

@@ -27,7 +27,7 @@
   "license": "ISC",
   "version": "0.1.0",
   "engines": {
-    "node": ">=12.3"
+    "node": ">=10"
   },
   "scripts": {
     "postversion": "npm publish --access public",

@vates/task/.USAGE.md

@@ -72,27 +72,6 @@ Task.warning(message, data)
 Task.set(property, value)
 ```
 
-### `Task.run([opts], fn, ...args)`
-
-This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
-
-```js
-// options can be passed
-await Task.run({ properties: { name: 'foo' } }, fn)
-
-// arguments can be passed
-await Task.run(fn, 'foo', 'bar')
-
-// context (this) is forwarded
-await Task.run.call(object, object.method)
-
-// as a convenience, you can pass a method name directly
-await Task.run.call(object, 'method')
-
-// everything together
-await Task.run.call(object, 'method', 'foo', 'bar')
-```
-
 ### `combineEvents`
 
 Create a consolidated log from individual events.
@@ -144,7 +123,7 @@ const onProgress = makeOnProgress({
   onTaskUpdate(taskLog) {},
 })
 
-Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
+Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
 ```
 
 It can also be fed event logs directly:

@vates/task/README.md

@@ -88,27 +88,6 @@ Task.warning(message, data)
 Task.set(property, value)
 ```
 
-### `Task.run([opts], fn, ...args)`
-
-This is a convenient shortcut to `new Task(opts).run(() => fn(...args))`.
-
-```js
-// options can be passed
-await Task.run({ properties: { name: 'foo' } }, fn)
-
-// arguments can be passed
-await Task.run(fn, 'foo', 'bar')
-
-// context (this) is forwarded
-await Task.run.call(object, object.method)
-
-// as a convenience, you can pass a method name directly
-await Task.run.call(object, 'method')
-
-// everything together
-await Task.run.call(object, 'method', 'foo', 'bar')
-```
-
 ### `combineEvents`
 
 Create a consolidated log from individual events.
@@ -132,7 +111,7 @@ const onProgress = makeOnProgress({
     // current status of the task as described in the previous section
     taskLog.status
 
-    // undefined or a dictionary of properties attached to the task
+    // undefined or a dictionnary of properties attached to the task
     taskLog.properties
 
     // timestamp at which the abortion was requested, undefined otherwise
@@ -160,7 +139,7 @@ const onProgress = makeOnProgress({
   onTaskUpdate(taskLog) {},
 })
 
-Task.run({ properties: { name: 'my task' }, onProgress }, asyncFn)
+Task.run({ data: { name: 'my task' }, onProgress }, asyncFn)
 ```
 
 It can also be fed event logs directly:

@vates/task/combineEvents.js

@@ -44,8 +44,6 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
       } else if (type === 'property') {
         ;(taskLog.properties ?? (taskLog.properties = { __proto__: null }))[event.name] = event.value
       } else if (type === 'end') {
-        taskLogs.delete(id)
-
         taskLog.end = event.timestamp
         taskLog.result = event.result
         taskLog.status = event.status

@vates/task/index.js

@@ -3,10 +3,6 @@
 const assert = require('node:assert').strict
 const { AsyncLocalStorage } = require('node:async_hooks')
 
-function alreadyEnded() {
-  throw new Error('task has already ended')
-}
-
 // define a read-only, non-enumerable, non-configurable property
 function define(object, property, value) {
   Object.defineProperty(object, property, { value })
@@ -25,7 +21,7 @@ const asyncStorage = global[asyncStorageKey] ?? (global[asyncStorageKey] = new A
 
 const getTask = () => asyncStorage.getStore()
 
-class Task {
+exports.Task = class Task {
   static get abortSignal() {
     const task = getTask()
     if (task !== undefined) {
@@ -33,25 +29,29 @@ class Task {
     }
   }
 
-  static run(...args) {
-    let opts = args[0]
-    let fn
-    if (typeof opts === 'object') {
-      args.shift()
-      fn = args[0]
-    } else {
-      fn = opts
-      opts = undefined
+  static info(message, data) {
+    const task = getTask()
+    if (task !== undefined) {
+      task.#emit('info', { data, message })
     }
-    args.shift()
+  }
 
-    const thisArg = this !== Task ? this : undefined
+  static run(opts, fn) {
+    return new this(opts).run(fn)
+  }
 
-    if (typeof fn !== 'function') {
-      fn = this[fn]
+  static set(name, value) {
+    const task = getTask()
+    if (task !== undefined) {
+      task.#emit('property', { name, value })
     }
+  }
 
-    return new Task(opts).run(() => fn.apply(thisArg, args))
+  static warning(message, data) {
+    const task = getTask()
+    if (task !== undefined) {
+      task.#emit('warning', { data, message })
+    }
   }
 
   static wrap(opts, fn) {
@@ -110,7 +110,9 @@ class Task {
         this.#emit('abortionRequested', { reason: signal.reason })
 
         if (!this.#running) {
-          this.#end(FAILURE, signal.reason)
+          const status = FAILURE
+          this.#status = status
+          this.#emit('end', { result: signal.reason, status })
         }
       }
     })
@@ -121,26 +123,12 @@ class Task {
   }
 
   #emit(type, data) {
-    assert.equal(this.#startData, undefined, 'task has not started yet')
-
     data.id = this.id
     data.timestamp = Date.now()
     data.type = type
     this.#onProgress(data)
   }
 
-  #end(status, result) {
-    assert.equal(this.#status, PENDING)
-
-    this.#status = status
-    this.#emit('end', { status, result })
-    this.#onProgress = alreadyEnded
-  }
-
-  info(message, data) {
-    this.#emit('info', { data, message })
-  }
-
   #maybeStart() {
     const startData = this.#startData
     if (startData !== undefined) {
@@ -152,7 +140,8 @@ class Task {
   async run(fn) {
     const result = await this.runInside(fn)
     if (this.status === PENDING) {
-      this.#end(SUCCESS, result)
+      this.#status = SUCCESS
+      this.#emit('end', { status: SUCCESS, result })
     }
     return result
   }
@@ -166,27 +155,18 @@ class Task {
     this.#maybeStart()
 
     try {
-      const result = await asyncStorage.run(this, fn, this)
+      const result = await asyncStorage.run(this, fn)
       this.#running = false
       return result
     } catch (result) {
-      this.#end(FAILURE, result)
+      const status = FAILURE
+
+      this.#status = status
+      this.#emit('end', { status, result })
       throw result
     }
   }
 
-  set(name, value) {
-    assert.equal(this.status, PENDING)
-
-    this.#emit('property', { name, value })
-  }
-
-  warning(message, data) {
-    assert.equal(this.status, PENDING)
-
-    this.#emit('warning', { data, message })
-  }
-
   wrap(fn) {
     const task = this
     return function taskRun() {
@@ -201,15 +181,3 @@ class Task {
     }
   }
 }
-exports.Task = Task
-
-// setup static aliases
-for (const name of ['info', 'set', 'warning']) {
-  const method = Task.prototype[name]
-  Task[name] = function () {
-    const task = getTask()
-    if (task !== undefined) {
-      return method.apply(task, arguments)
-    }
-  }
-}

@vates/task/index.test.js

@@ -3,7 +3,6 @@
 const assert = require('node:assert').strict
 const { describe, it } = require('test')
 
-const { makeOnProgress } = require('./combineEvents.js')
 const { Task } = require('./index.js')
 
 const noop = Function.prototype
@@ -163,44 +162,6 @@ describe('Task', function () {
     })
   })
 
-  describe('.run()', function () {
-    function fn(...args) {
-      Task.set('baz', 'qux')
-
-      return [this, ...args]
-    }
-    const obj = { fn }
-
-    it('accepts optional options', async function () {
-      assert.deepEqual(
-        await Task.run(
-          {
-            properties: { foo: 'bar' },
-            onProgress: makeOnProgress({
-              onRootTaskEnd(log) {
-                assert.deepEqual(log.properties, { __proto__: null, foo: 'bar', baz: 'qux' })
-              },
-            }),
-          },
-          fn
-        ),
-        [undefined]
-      )
-    })
-
-    it('accepts optional arguments', async function () {
-      assert.deepEqual(await Task.run(fn, 'foo', 'bar'), [undefined, 'foo', 'bar'])
-    })
-
-    it('accepts optional context', async function () {
-      assert.deepEqual(await Task.run.call('foo', fn), ['foo'])
-    })
-
-    it('accepts a method name instead of a function', async function () {
-      assert.deepEqual(await Task.run.call(obj, 'fn'), [obj])
-    })
-  })
-
   describe('.set()', function () {
     it('does nothing when run outside a task', function () {
       Task.set('progress', 10)
@@ -219,31 +180,6 @@ describe('Task', function () {
     })
   })
 
-  describe('#set()', function () {
-    const name = 'progress'
-    const value = 10
-
-    it('throws when the task is not started', function () {
-      const task = createTask()
-      assert.throws(() => task.set(name, value), { message: 'task has not started yet' })
-    })
-
-    it(`emits an property message`, async function () {
-      const task = createTask()
-      await task.run(async () => {
-        await Task.run(() => {
-          task.set(name, value)
-
-          assertEvent(task, {
-            name,
-            type: 'property',
-            value,
-          })
-        })
-      })
-    })
-  })
-
   describe('.warning()', function () {
     it('does nothing when run outside a task', function () {
       Task.warning('foo')
@@ -262,30 +198,6 @@ describe('Task', function () {
     })
   })
 
-  for (const type of ['info', 'warning']) {
-    describe(`#${type}()`, function () {
-      it('throws when the task is not started', function () {
-        const task = createTask()
-        assert.throws(() => task[type]('foo'), { message: 'task has not started yet' })
-      })
-
-      it(`emits an ${type} message`, async function () {
-        const task = createTask()
-        await task.run(async () => {
-          await Task.run(() => {
-            task[type]('foo')
-
-            assertEvent(task, {
-              data: undefined,
-              message: 'foo',
-              type,
-            })
-          })
-        })
-      })
-    })
-  }
-
   describe('#id', function () {
     it('can be set', function () {
       const task = createTask()

@vates/task/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.4.1",
+  "version": "0.2.0",
   "engines": {
     "node": ">=14"
   },

@vates/xml-rpc/.USAGE.md

@@ -1,128 +0,0 @@
-### Formatting
-
-```js
-import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
-
-// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
-xmlRpcFormatter.format_methodCall({
-  methodName: 'examples.getStateName',
-  params: [40],
-})
-// → {
-//   name: 'methodCall',
-//   children: [
-//     {
-//       name: 'methodName',
-//       children: [ 'examples.getStateName' ]
-//     },
-//     {
-//       name: 'params',
-//       children: [
-//         {
-//           name: 'param',
-//           children: [ { name: 'value', children: [ '40' ] } ]
-//         }
-//       ]
-//     }
-//   ]
-// }
-
-// There is a dedicated method to format a JS value
-xmlRpcFormatter.format_js(new Date(900684535000))
-// → {
-//   name: 'dateTime.iso8601',
-//   children: [ '1998-07-17T14:08:55.000Z' ]
-// }
-```
-
-Formatting XML is out of this library's scope, but you can achieve it easily with
-`@vates/xml`:
-
-```js
-import { formatXml } from '@vates/xml/format'
-
-// it can now be passed to the XML-RPC parser
-const tree = xmlRpcFormatter.format_methodCall({
-  methodName: 'examples.getStateName',
-  params: [40],
-})
-
-// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
-const xml = formatXml(tree, { indent: 0 })
-```
-
-### Parsing
-
-```js
-import { xmlRpcParser } from '@vates/xml-rpc/parser'
-
-xmlRpcParser.parse({
-  name: 'methodResponse',
-  children: [
-    {
-      name: 'params',
-      children: [
-        {
-          name: 'param',
-          children: [
-            {
-              name: 'value',
-              children: [
-                {
-                  name: 'string',
-                  children: ['South Dakota'],
-                },
-              ],
-            },
-          ],
-        },
-      ],
-    },
-  ],
-})
-// { params: [ 'South Dakota' ] }
-
-// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
-xmlRpcParser.parse_array({
-  name: 'array',
-  children: [
-    {
-      name: 'data',
-      children: [
-        {
-          name: 'value',
-          children: [{ name: 'boolean', children: ['1'] }],
-        },
-        {
-          name: 'value',
-          children: [{ name: 'int', children: ['42'] }],
-        },
-      ],
-    },
-  ],
-})
-// [ true, 42 ]
-```
-
-Parsing XML is out of this library's scope, but you can achieve it easily with
-`@vates/xml`:
-
-```js
-import { parseXml } from '@vates/xml/parse'
-
-const xml = `
-<?xml version="1.0"?>
-<methodResponse>
-  <params>
-    <param>
-        <value><string>South Dakota</string></value>
-    </param>
-  </params>
-</methodResponse>
-`
-
-const tree = parseXml(xml)
-
-// it can now be passed to the XML-RPC parser
-xmlRpcParser.parse(tree)
-```

@vates/xml-rpc/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/xml-rpc/README.md

@@ -1,161 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/xml-rpc
-
-[![Package Version](https://badgen.net/npm/v/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc) ![License](https://badgen.net/npm/license/@vates/xml-rpc) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml-rpc)](https://bundlephobia.com/result?p=@vates/xml-rpc) [![Node compatibility](https://badgen.net/npm/node/@vates/xml-rpc)](https://npmjs.org/package/@vates/xml-rpc)
-
-> Parse and format XML-RPC messages, do not handle transport or XML
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/xml-rpc):
-
-```sh
-npm install --save @vates/xml-rpc
-```
-
-## Usage
-
-### Formatting
-
-```js
-import { xmlRpcFormatter } from '@vates/xml-rpc/formatter'
-
-// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
-xmlRpcFormatter.format_methodCall({
-  methodName: 'examples.getStateName',
-  params: [40],
-})
-// → {
-//   name: 'methodCall',
-//   children: [
-//     {
-//       name: 'methodName',
-//       children: [ 'examples.getStateName' ]
-//     },
-//     {
-//       name: 'params',
-//       children: [
-//         {
-//           name: 'param',
-//           children: [ { name: 'value', children: [ '40' ] } ]
-//         }
-//       ]
-//     }
-//   ]
-// }
-
-// There is a dedicated method to format a JS value
-xmlRpcFormatter.format_js(new Date(900684535000))
-// → {
-//   name: 'dateTime.iso8601',
-//   children: [ '1998-07-17T14:08:55.000Z' ]
-// }
-```
-
-Formatting XML is out of this library's scope, but you can achieve it easily with
-`@vates/xml`:
-
-```js
-import { formatXml } from '@vates/xml/format'
-
-// it can now be passed to the XML-RPC parser
-const tree = xmlRpcFormatter.format_methodCall({
-  methodName: 'examples.getStateName',
-  params: [40],
-})
-
-// avoid any indentation or new lines as it will confuse many XML-RPC parsers.
-const xml = formatXml(tree, { indent: 0 })
-```
-
-### Parsing
-
-```js
-import { xmlRpcParser } from '@vates/xml-rpc/parser'
-
-xmlRpcParser.parse({
-  name: 'methodResponse',
-  children: [
-    {
-      name: 'params',
-      children: [
-        {
-          name: 'param',
-          children: [
-            {
-              name: 'value',
-              children: [
-                {
-                  name: 'string',
-                  children: ['South Dakota'],
-                },
-              ],
-            },
-          ],
-        },
-      ],
-    },
-  ],
-})
-// { params: [ 'South Dakota' ] }
-
-// There is a dedicated method per XML-RPC tag (`array`, `string`, etc.)
-xmlRpcParser.parse_array({
-  name: 'array',
-  children: [
-    {
-      name: 'data',
-      children: [
-        {
-          name: 'value',
-          children: [{ name: 'boolean', children: ['1'] }],
-        },
-        {
-          name: 'value',
-          children: [{ name: 'int', children: ['42'] }],
-        },
-      ],
-    },
-  ],
-})
-// [ true, 42 ]
-```
-
-Parsing XML is out of this library's scope, but you can achieve it easily with
-`@vates/xml`:
-
-```js
-import { parseXml } from '@vates/xml/parse'
-
-const xml = `
-<?xml version="1.0"?>
-<methodResponse>
-  <params>
-    <param>
-        <value><string>South Dakota</string></value>
-    </param>
-  </params>
-</methodResponse>
-`
-
-const tree = parseXml(xml)
-
-// it can now be passed to the XML-RPC parser
-xmlRpcParser.parse(tree)
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/xml-rpc/formatter.js

@@ -1,104 +0,0 @@
-'use strict'
-
-function node(name, attributes, children) {
-  if (Array.isArray(attributes)) {
-    children = attributes
-    attributes = undefined
-  }
-
-  return { name, attributes, children }
-}
-
-class XmlRpcFormatter {
-  format_js(value) {
-    const type = typeof value
-    if (type === 'object') {
-      return value === null
-        ? this.format_nil()
-        : value instanceof Date
-          ? this['format_dateTime.iso8601'](value)
-          : Array.isArray(value)
-            ? this.format_array(value)
-            : Buffer.isBuffer(value)
-              ? this.format_base64(value)
-              : this.format_struct(value)
-    }
-
-    return this['format_' + type](value)
-  }
-
-  format_array(values) {
-    return node('array', [node('data', values.map(this.format_value, this))])
-  }
-
-  format_base64(buf) {
-    return node('base64', [buf.toString('base64')])
-  }
-
-  format_boolean(bool) {
-    return node('boolean', [bool ? '1' : '0'])
-  }
-
-  'format_dateTime.iso8601'(date) {
-    return node('dateTime.iso8601', [date.toISOString()])
-  }
-
-  format_double(str) {
-    return node('double', [String(str)])
-  }
-
-  format_fault(val) {
-    return node('fault', [this.format_value(val)])
-  }
-
-  format_i4(str) {
-    return node('i4', [String(str)])
-  }
-
-  format_int(str) {
-    return node('int', [String(str)])
-  }
-
-  format_member([name, value]) {
-    return node('member', [node('name', [name]), this.format_value(value)])
-  }
-
-  format_methodCall({ methodName, params }) {
-    return node('methodCall', [node('methodName', [methodName]), this.format_params(params)])
-  }
-
-  format_methodResponse({ fault, params }) {
-    return node('methodResponse', [fault !== undefined ? this.format_fault(fault) : this.format_params(params)])
-  }
-
-  format_nil() {
-    return node('nil')
-  }
-
-  format_number(num) {
-    return Number.isInteger(num) ? this.format_int(num) : this.format_double(num)
-  }
-
-  format_param(value) {
-    return node('param', [this.format_value(value)])
-  }
-
-  format_params(params) {
-    return node('params', params.map(this.format_param, this))
-  }
-
-  format_string(str) {
-    return str
-  }
-
-  format_struct(struct) {
-    return node('struct', Object.entries(struct).map(this.format_member, this))
-  }
-
-  format_value(value) {
-    return node('value', [this.format_js(value)])
-  }
-}
-exports.Formatter = XmlRpcFormatter
-
-exports.xmlRpcFormatter = new XmlRpcFormatter()

@vates/xml-rpc/package.json

@@ -1,33 +0,0 @@
-{
-  "private": false,
-  "name": "@vates/xml-rpc",
-  "description": "Parse and format XML-RPC messages, do not handle transport or XML",
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml-rpc",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/xml-rpc",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "1.0.0",
-  "engines": {
-    "node": ">=12"
-  },
-  "devDependencies": {
-    "@vates/xml": "^2.0.0",
-    "test": "^3.3.0"
-  },
-  "exports": {
-    "./formatter": "./formatter.js",
-    "./parser": "./parser.js"
-  },
-  "scripts": {
-    "postversion": "npm publish --access public",
-    "test": "node--test"
-  }
-}

@vates/xml-rpc/parser.js

@@ -1,133 +0,0 @@
-'use strict'
-
-function toValue(node) {
-  const obj = { __proto__: null }
-  for (const child of node.children) {
-    if (typeof child === 'string') {
-      return child
-    }
-
-    const { name } = child
-    if (Object.hasOwn(obj, name)) {
-      const prev = obj[name]
-      if (Array.isArray(obj, name)) {
-        prev.push(child)
-      } else {
-        obj[name] = [prev, child]
-      }
-    } else {
-      obj[name] = child
-    }
-  }
-  return obj
-}
-
-// https://en.wikipedia.org/wiki/XML-RPC#Data_types
-class XmlRpcParser {
-  parse(node) {
-    return this['parse_' + node.name](node)
-  }
-
-  parse_array({ children: [data] }) {
-    return data.children.map(this.parse_value, this)
-  }
-
-  parse_base64({ children: [str] }) {
-    return Buffer.from(str, 'base64')
-  }
-
-  parse_boolean({ children: [str] }) {
-    return Boolean(Number(str))
-  }
-
-  'parse_dateTime.iso8601'({ children: [str] }) {
-    const date = new Date(str)
-
-    // date successfully parsed
-    if (!Number.isNaN(+date)) {
-      return date
-    }
-
-    // XML-RPC dates may use YYYYMMDD instead of YYYY-MM-DD which ECMAScript
-    // does not parse
-    return new Date(str.slice(0, 4) + '-' + str.slice(4, 6) + '-' + str.slice(6))
-  }
-
-  parse_double({ children: [str] }) {
-    return Number(str)
-  }
-
-  parse_fault({ children: [value] }) {
-    return this.parse_value(value)
-  }
-
-  parse_i4({ children: [str] }) {
-    return Number(str)
-  }
-
-  parse_int({ children: [str] }) {
-    return Number(str)
-  }
-
-  parse_member(member) {
-    member = toValue(member)
-    return [member.name.children[0], this.parse_value(member.value)]
-  }
-
-  parse_methodCall(methodCall) {
-    methodCall = toValue(methodCall)
-    return {
-      methodName: methodCall.methodName.children[0],
-      params: this.parse_params(methodCall.params),
-    }
-  }
-
-  parse_methodResponse(methodResponse) {
-    methodResponse = toValue(methodResponse)
-
-    const { fault } = methodResponse
-    if (fault !== undefined) {
-      return { fault: this.parse_fault(fault) }
-    }
-
-    return { params: this.parse_params(methodResponse.params) }
-  }
-
-  parse_nil() {
-    return null
-  }
-
-  parse_param(param) {
-    return this.parse_value(param.children[0])
-  }
-
-  parse_params(params) {
-    return params.children.map(this.parse_param, this)
-  }
-
-  parse_string({ children: [str] }) {
-    return str
-  }
-
-  parse_struct(struct) {
-    return Object.fromEntries(struct.children.map(this.parse_member, this))
-  }
-
-  parse_value({ children }) {
-    if (children.length === 0) {
-      return ''
-    }
-
-    const child = children[0]
-
-    // can be a plain string
-    if (typeof child === 'string') {
-      return child
-    }
-
-    return this.parse(child)
-  }
-}
-exports.XmlRpcParser = XmlRpcParser
-
-exports.xmlRpcParser = new XmlRpcParser()

@vates/xml-rpc/test/formatter.js

@@ -1,19 +0,0 @@
-'use strict'
-
-const { formatXml } = require('@vates/xml/format')
-const { join } = require('node:path')
-const { parseXml } = require('@vates/xml/parse')
-const { readFileSync } = require('node:fs')
-const { xmlRpcFormatter } = require('@vates/xml-rpc/formatter')
-const assert = require('node:assert/strict')
-const test = require('test')
-
-const methodCall = require('./methodCall.js')
-
-test(function () {
-  const xmlTree = parseXml(readFileSync(join(__dirname, 'methodCall.xml')))
-
-  const xmlRpc = xmlRpcFormatter.format_methodCall(methodCall)
-
-  assert.deepEqual(parseXml(formatXml(xmlRpc)), xmlTree)
-})

@vates/xml-rpc/test/methodCall.js

@@ -1,16 +0,0 @@
-'use strict'
-
-module.exports = {
-  methodName: 'examples.getStateName',
-  params: [
-    [1404, 'Something here'],
-    Buffer.from([121, 111, 117, 32, 99, 97, 110, 39, 116, 32, 114, 101, 97, 100, 32, 116, 104, 105, 115, 33]),
-    true,
-    new Date('1998-07-17T14:08:55.000Z'),
-    -12.53,
-    42,
-    'Hello world!',
-    { foo: 1, bar: 2 },
-    null,
-  ],
-}

@vates/xml-rpc/test/methodCall.xml

@@ -1,73 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<methodCall>
-  <methodName>examples.getStateName</methodName>
-  <params>
-    <param>
-      <value>
-        <array>
-          <data>
-            <value>
-              <int>1404</int>
-            </value>
-            <value>
-              Something here
-            </value>
-          </data>
-        </array>
-      </value>
-    </param>
-    <param>
-      <value>
-        <base64>eW91IGNhbid0IHJlYWQgdGhpcyE=</base64>
-      </value>
-    </param>
-    <param>
-      <value>
-        <boolean>1</boolean>
-      </value>
-    </param>
-    <param>
-      <value>
-        <dateTime.iso8601>1998-07-17T14:08:55.000Z</dateTime.iso8601>
-      </value>
-    </param>
-    <param>
-      <value>
-        <double>-12.53</double>
-      </value>
-    </param>
-    <param>
-      <value>
-        <int>42</int>
-      </value>
-    </param>
-    <param>
-      <value>
-        Hello world!
-      </value>
-    </param>
-    <param>
-      <value>
-        <struct>
-          <member>
-            <name>foo</name>
-            <value>
-              <int>1</int>
-            </value>
-          </member>
-          <member>
-            <name>bar</name>
-            <value>
-              <int>2</int>
-            </value>
-          </member>
-        </struct>
-      </value>
-    </param>
-    <param>
-      <value>
-        <nil />
-      </value>
-    </param>
-  </params>
-</methodCall>

@vates/xml-rpc/test/parser.js

@@ -1,32 +0,0 @@
-'use strict'
-
-const { join } = require('node:path')
-const { parseXml } = require('@vates/xml/parse')
-const { readFileSync } = require('node:fs')
-const { xmlRpcParser } = require('@vates/xml-rpc/parser')
-const assert = require('node:assert/strict')
-const test = require('test')
-
-const methodCall = require('./methodCall.js')
-
-test('parser.parse()', function () {
-  const xml = String(readFileSync(join(__dirname, 'methodCall.xml')))
-  const xmlRpc = parseXml(xml)
-
-  assert.deepEqual(xmlRpcParser.parse(xmlRpc), methodCall)
-})
-
-test('datetime without dashes nor milliseconds', function () {
-  assert.deepEqual(
-    xmlRpcParser.parse({ name: 'dateTime.iso8601', children: ['19980717T14:08:55Z'] }),
-    new Date('1998-07-17T14:08:55Z')
-  )
-})
-
-test('i4 support', function () {
-  assert.equal(xmlRpcParser.parse({ name: 'i4', children: ['42'] }), 42)
-})
-
-test('string support', function () {
-  assert.equal(xmlRpcParser.parse({ name: 'string', children: ['foo'] }), 'foo')
-})

@vates/xml/.USAGE.md

@@ -1,56 +0,0 @@
-### `parseXml(xml, [opts])`
-
-> Based on [`sax`](https://www.npmjs.com/package/sax)
-
-`opts` is an optional object which can contain the following options:
-
-- `normalize = true`: if true, then turn any whitespace into a single space
-- `strict = true`: whether or not to be a jerk
-- `trim = true`: whether or not to trim text nodes
-
-```js
-import { parseXml } from '@xen-orchestra/xml/parse'
-
-const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
-<tag1 attr1="value1" attr2="value2">
-  Text &amp; entities
-  <tag2 />
-  <tag2 />
-  <ns:tag3 />
-</tag1>
-`)
-// → {
-//   name: 'tag1',
-//   attributes: { attr1: 'value1', attr2: 'value2' },
-//   children: [
-//     'Text & entities',
-//     { name: 'tag2', attributes: {}, children: [] },
-//     { name: 'tag2', attributes: {}, children: [] },
-//     { name: 'ns:tag3', attributes: {}, children: [] }
-//   ]
-// }
-```
-
-### `formatXml(tree, [opts])`
-
-`opts` is an optional object which can contain the following options:
-
-- `includeDeclaration = true`: whether to include an XML declaration
-- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
-
-```js
-import { formatXml } from '@xen-orchestra/xml/format'
-
-formatXml({
-  name: 'tag1',
-  attributes: { attr1: 'value1', attr2: 'value2' },
-  children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
-})
-// → <?xml version="1.0" encoding="UTF-8"?>
-// <tag1 attr1="value1" attr2="value2">
-//   Text &amp; entities
-//   <tag2 />
-//   <tag2 />
-//   <ns:tag3 />
-// </tag1>
-```

@vates/xml/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/xml/README.md

@@ -1,89 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/xml
-
-[![Package Version](https://badgen.net/npm/v/@vates/xml)](https://npmjs.org/package/@vates/xml) ![License](https://badgen.net/npm/license/@vates/xml) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/xml)](https://bundlephobia.com/result?p=@vates/xml) [![Node compatibility](https://badgen.net/npm/node/@vates/xml)](https://npmjs.org/package/@vates/xml)
-
-> Simple XML formatting/parsing
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/xml):
-
-```sh
-npm install --save @vates/xml
-```
-
-## Usage
-
-### `parseXml(xml, [opts])`
-
-> Based on [`sax`](https://www.npmjs.com/package/sax)
-
-`opts` is an optional object which can contain the following options:
-
-- `normalize = true`: if true, then turn any whitespace into a single space
-- `strict = true`: whether or not to be a jerk
-- `trim = true`: whether or not to trim text nodes
-
-```js
-import { parseXml } from '@xen-orchestra/xml/parse'
-
-const tree = parseXml(`<?xml version="1.0" encoding="UTF-8"?>
-<tag1 attr1="value1" attr2="value2">
-  Text &amp; entities
-  <tag2 />
-  <tag2 />
-  <ns:tag3 />
-</tag1>
-`)
-// → {
-//   name: 'tag1',
-//   attributes: { attr1: 'value1', attr2: 'value2' },
-//   children: [
-//     'Text & entities',
-//     { name: 'tag2', attributes: {}, children: [] },
-//     { name: 'tag2', attributes: {}, children: [] },
-//     { name: 'ns:tag3', attributes: {}, children: [] }
-//   ]
-// }
-```
-
-### `formatXml(tree, [opts])`
-
-`opts` is an optional object which can contain the following options:
-
-- `includeDeclaration = true`: whether to include an XML declaration
-- `indent = 2`: string or number of spaces to use to indent; if an empty string or `0`, no indent or new lines will be used
-
-```js
-import { formatXml } from '@xen-orchestra/xml/format'
-
-formatXml({
-  name: 'tag1',
-  attributes: { attr1: 'value1', attr2: 'value2' },
-  children: ['Text & entities', { name: 'tag2' }, { name: 'tag2' }, { name: 'ns:tag3' }],
-})
-// → <?xml version="1.0" encoding="UTF-8"?>
-// <tag1 attr1="value1" attr2="value2">
-//   Text &amp; entities
-//   <tag2 />
-//   <tag2 />
-//   <ns:tag3 />
-// </tag1>
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/xml/cli.js

@@ -1,30 +0,0 @@
-#!/usr/bin/env node
-
-'use strict'
-
-const { inspect } = require('node:util')
-const { formatXml } = require('@vates/xml/format')
-const { parseXml } = require('@vates/xml/parse')
-const { readFileSync } = require('node:fs')
-
-function log(val) {
-  process.stdout.write(inspect(val, false, null, true))
-  process.stdout.write('\n')
-}
-
-function main([inputPath = 0]) {
-  const input = readFileSync(inputPath)
-
-  // attempt to parse from JSON
-  let data
-  try {
-    data = JSON.parse(input)
-  } catch (error) {
-    // fallback to XML
-    log(parseXml(input))
-    return
-  }
-
-  process.stdout.write(formatXml(data))
-}
-main(process.argv.slice(2))

@vates/xml/format.js

@@ -1,64 +0,0 @@
-'use strict'
-
-const ENTITIES = {
-  '"': '"',
-  '&': '&',
-  '<': '&lt;',
-  '>': '&gt;',
-  "'": '&apos;',
-}
-const RE = new RegExp(Object.keys(ENTITIES).join('|'), 'g')
-function replace(match) {
-  return ENTITIES[match]
-}
-function escape(str) {
-  return str.replace(RE, replace)
-}
-
-function formatNode(node, depth) {
-  const indent = this.indent.repeat(depth)
-
-  if (typeof node === 'object') {
-    const line = [indent, '<', node.name]
-    const { attributes } = node
-    if (attributes !== undefined) {
-      for (const name of Object.keys(attributes)) {
-        line.push(' ', name, '="', escape(attributes[name]), '"')
-      }
-    }
-
-    const { children } = node
-    if (children === undefined || children.length === 0) {
-      line.push(' />')
-      this.lines.push(line.join(''))
-    } else {
-      line.push('>')
-      this.lines.push(line.join(''))
-
-      for (const child of children) {
-        formatNode.call(this, child, depth + 1)
-      }
-
-      this.lines.push(indent + '</' + node.name + '>')
-    }
-  } else {
-    // string (or scalar)
-    this.lines.push(indent + escape(String(node)))
-  }
-}
-
-exports.formatXml = function formatXml(tree, { includeDeclaration = true, indent = 2 } = {}) {
-  const lines = []
-
-  if (includeDeclaration) {
-    lines.push('<?xml version="1.0" encoding="UTF-8"?>')
-  }
-
-  const ctx = {
-    lines,
-    indent: typeof indent === 'number' ? ' '.repeat(indent) : indent,
-  }
-  formatNode.call(ctx, tree, 0)
-
-  return lines.join(indent !== 0 && indent !== '' ? '\n' : '')
-}

@vates/xml/package.json

@@ -1,35 +0,0 @@
-{
-  "private": false,
-  "name": "@vates/xml",
-  "description": "Simple XML formatting/parsing",
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/xml",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/xml",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "2.0.0",
-  "engines": {
-    "node": ">=14"
-  },
-  "dependencies": {
-    "sax": "^1.3.0"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
-  "exports": {
-    "./format": "./format.js",
-    "./parse": "./parse.js"
-  },
-  "scripts": {
-    "postversion": "npm publish --access public",
-    "test": "node--test"
-  }
-}

@vates/xml/parse.js

@@ -1,23 +0,0 @@
-'use strict'
-
-const sax = require('sax')
-
-exports.parseXml = function parseXml(xml, { normalize = true, strict = true, trim = true } = {}) {
-  const stack = [{ children: [] }]
-
-  const parser = sax.parser(strict, { normalize, trim })
-  parser.ontext = text => {
-    stack[stack.length - 1].children.push(text)
-  }
-  parser.onopentag = ({ name, attributes }) => {
-    stack.push({ name, attributes, children: [] })
-  }
-  parser.onclosetag = () => {
-    const node = stack.pop()
-    stack[stack.length - 1].children.push(node)
-  }
-
-  parser.write(xml).close()
-
-  return stack[0].children[0]
-}

@vates/xml/test/data.json

@@ -1,25 +0,0 @@
-{
-  "name": "tag1",
-  "attributes": {
-    "attr1": "value1",
-    "attr2": "value2"
-  },
-  "children": [
-    "Text & entities",
-    {
-      "name": "tag2",
-      "attributes": {},
-      "children": []
-    },
-    {
-      "name": "tag2",
-      "attributes": {},
-      "children": []
-    },
-    {
-      "name": "ns:tag3",
-      "attributes": {},
-      "children": []
-    }
-  ]
-}

@vates/xml/test/data.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<tag1 attr1="value1" attr2="value2">
-  Text &amp; entities
-  <tag2 />
-  <tag2 />
-  <ns:tag3 />
-</tag1>

@vates/xml/test/format.js

@@ -1,18 +0,0 @@
-'use strict'
-
-const { formatXml } = require('@vates/xml/format')
-const { join } = require('node:path')
-const { readFileSync } = require('node:fs')
-const assert = require('node:assert/strict')
-const test = require('test')
-
-test('format()', function () {
-  const xml = String(readFileSync(join(__dirname, 'data.xml')))
-  const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
-
-  assert.equal(formatXml(tree) + '\n', xml)
-})
-
-test('supports missing attributes and children', function () {
-  assert.equal(formatXml({ name: 'foo' }, { includeDeclaration: false }), '<foo />')
-})

@vates/xml/test/parse.js

@@ -1,14 +0,0 @@
-'use strict'
-
-const { parseXml } = require('@vates/xml/parse')
-const { readFileSync } = require('node:fs')
-const { join } = require('node:path')
-const assert = require('node:assert/strict')
-const test = require('test')
-
-test('parse()', function () {
-  const xml = String(readFileSync(join(__dirname, 'data.xml')))
-  const tree = JSON.parse(readFileSync(join(__dirname, 'data.json')))
-
-  assert.deepEqual(parseXml(xml), tree)
-})

@xen-orchestra/async-map/package.json

@@ -35,7 +35,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
+    "sinon": "^15.0.1",
     "test": "^3.2.1"
   }
 }

@xen-orchestra/audit-core/index.js

@@ -130,10 +130,6 @@ class AuditCore {
     return nValid
   }
 
-  get(id) {
-    return this._storage.get(id)
-  }
-
   async *getFrom(newest) {
     const storage = this._storage
 

@xen-orchestra/audit-core/index.spec.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { afterEach, describe, it } = require('test')
+const { afterEach, describe, it } = require('tap').mocha
 
 const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')
 

@xen-orchestra/audit-core/package.json

@@ -7,16 +7,16 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.3.0",
+  "version": "0.2.3",
   "engines": {
     "node": ">=14"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "tap --lines 67 --functions 92 --branches 52 --statements 67"
   },
   "dependencies": {
-    "@vates/decorate-with": "^2.1.0",
+    "@vates/decorate-with": "^2.0.0",
     "@xen-orchestra/log": "^0.6.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"
@@ -28,6 +28,6 @@
     "url": "https://vates.fr"
   },
   "devDependencies": {
-    "test": "^3.3.0"
+    "tap": "^16.0.1"
   }
 }

@xen-orchestra/backups-cli/package.json

@@ -7,9 +7,9 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.54.0",
-    "@xen-orchestra/fs": "^4.1.7",
-    "filenamify": "^6.0.0",
+    "@xen-orchestra/backups": "^0.39.0",
+    "@xen-orchestra/fs": "^4.0.1",
+    "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0"
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.0.25",
+  "version": "1.0.9",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/HealthCheckVmBackup.mjs

@@ -49,25 +49,26 @@ export class HealthCheckVmBackup {
         }
 
         // wait for the 'Running' event to be really stored in local xapi object cache
-
         restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
           timeout: remainingTimeout,
-          timeoutMessage: refOrUuid =>
-            `local xapi  did not get Running state for VM ${refOrUuid} after ${timeout / 1000} second`,
         })
 
         const running = new Date()
         remainingTimeout -= running - started
 
+        if (remainingTimeout < 0) {
+          throw new Error(`local xapi  did not get Running state for VM ${restoredId} after ${timeout / 1000} second`)
+        }
         // wait for the guest tool version to be defined
         await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
           timeout: remainingTimeout,
-          timeoutMessage: refOrUuid =>
-            `timeout reached while waiting for ${refOrUuid} to report the driver version through the Xen tools. Please check or update the Xen tools.`,
         })
 
         const guestToolsReady = new Date()
         remainingTimeout -= guestToolsReady - running
+        if (remainingTimeout < 0) {
+          throw new Error(`local xapi  did not get he guest tools check ${restoredId} after ${timeout / 1000} second`)
+        }
 
         if (waitForScript) {
           const startedRestoredVm = await xapi.waitObjectState(
@@ -78,10 +79,19 @@ export class HealthCheckVmBackup {
                 vm.xenstore_data['vm-data/xo-backup-health-check'] === 'failure'),
             {
               timeout: remainingTimeout,
-              timeoutMessage: refOrUuid =>
-                `timeout reached while waiting for ${refOrUuid} to report the startup script execution.`,
             }
           )
+          const scriptOk = new Date()
+          remainingTimeout -= scriptOk - guestToolsReady
+          if (remainingTimeout < 0) {
+            throw new Error(
+              `Backup health check script did not update vm-data/xo-backup-health-check of ${restoredId} after ${
+                timeout / 1000
+              } second, got ${
+                startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check']
+              } instead of 'success' or 'failure'`
+            )
+          }
 
           if (startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check'] !== 'success') {
             const message = startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check-error']

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -4,229 +4,23 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { importIncrementalVm } from './_incrementalVm.mjs'
 import { Task } from './Task.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
-import { VhdNegative, VhdSynthetic } from 'vhd-lib'
-import { decorateClass } from '@vates/decorate-with'
-import { createLogger } from '@xen-orchestra/log'
-import { dirname, join } from 'node:path'
-import pickBy from 'lodash/pickBy.js'
-import { defer } from 'golike-defer'
 
-const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
-async function resolveUuid(xapi, cache, uuid, type) {
-  if (uuid == null) {
-    return uuid
-  }
-  const ref = cache.get(uuid)
-  if (ref === undefined) {
-    cache.set(uuid, xapi.call(`${type}.get_by_uuid`, uuid))
-  }
-  return cache.get(uuid)
-}
 export class ImportVmBackup {
-  constructor({
-    adapter,
-    metadata,
-    srUuid,
-    xapi,
-    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
-  }) {
+  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs = {} } = {} }) {
     this._adapter = adapter
-    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
+    this._importIncrementalVmSettings = { newMacAddresses, mapVdisSrs }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
   }
 
-  async #getPathOfVdiSnapshot(snapshotUuid) {
-    const metadata = this._metadata
-    if (this._pathToVdis === undefined) {
-      const backups = await this._adapter.listVmBackups(
-        this._metadata.vm.uuid,
-        ({ mode, timestamp }) => mode === 'delta' && timestamp >= metadata.timestamp
-      )
-      const map = new Map()
-      for (const backup of backups) {
-        for (const [vdiRef, vdi] of Object.entries(backup.vdis)) {
-          map.set(vdi.uuid, backup.vhds[vdiRef])
-        }
-      }
-      this._pathToVdis = map
-    }
-    return this._pathToVdis.get(snapshotUuid)
-  }
-
-  async _reuseNearestSnapshot($defer, ignoredVdis) {
-    const metadata = this._metadata
-    const { mapVdisSrs } = this._importIncrementalVmSettings
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
-    const streams = {}
-    const metdataDir = dirname(metadata._filename)
-    const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
-
-    for (const [vdiRef, vdi] of Object.entries(vdis)) {
-      const vhdPath = join(metdataDir, vhds[vdiRef])
-
-      let xapiDisk
-      try {
-        xapiDisk = await this._xapi.getRecordByUuid('VDI', vdi.$snapshot_of$uuid)
-      } catch (err) {
-        // if this disk is not present anymore, fall back to default restore
-        warn(err)
-      }
-
-      let snapshotCandidate, backupCandidate
-      if (xapiDisk !== undefined) {
-        debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
-        for (const snapshotRef of xapiDisk.snapshots) {
-          const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
-          debug('handling snapshot', { snapshot })
-
-          // take only the first snapshot
-          if (snapshotCandidate && snapshotCandidate.snapshot_time < snapshot.snapshot_time) {
-            debug('already got a better candidate')
-            continue
-          }
-
-          // have a corresponding backup more recent than metadata ?
-          const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
-          if (pathToSnapshotData === undefined) {
-            debug('no backup linked to this snaphot')
-            continue
-          }
-          if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
-            debug('not restored on the same SR', { snapshotSr: snapshot.$SR.uuid, mapVdisSrs, srUuid: this._srUuid })
-            continue
-          }
-
-          debug('got a candidate', pathToSnapshotData)
-
-          snapshotCandidate = snapshot
-          backupCandidate = pathToSnapshotData
-        }
-      }
-
-      let stream
-      const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
-      if (vhdPath === backupWithSnapshotPath) {
-        // all the data are already on the host
-        debug('direct reuse of a snapshot')
-        stream = null
-        vdis[vdiRef].baseVdi = snapshotCandidate
-        // go next disk , we won't use this stream
-        continue
-      }
-
-      let disposableDescendants
-
-      const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
-
-      // this will also clean if another disk of this VM backup fails
-      // if user really only need to restore non failing disks he can retry with ignoredVdis
-      let disposed = false
-      const disposeOnce = async () => {
-        if (!disposed) {
-          disposed = true
-          try {
-            await disposableDescendants?.dispose()
-            await disposableSynthetic?.dispose()
-          } catch (error) {
-            warn('openVhd: failed to dispose VHDs', { error })
-          }
-        }
-      }
-      $defer.onFailure(() => disposeOnce())
-
-      const parentVhd = disposableSynthetic.value
-      await parentVhd.readBlockAllocationTable()
-      debug('got vhd synthetic of parents', parentVhd.length)
-
-      if (snapshotCandidate !== undefined) {
-        try {
-          debug('will try to use differential restore', {
-            backupWithSnapshotPath,
-            vhdPath,
-            vdiRef,
-          })
-
-          disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
-            until: vhdPath,
-          })
-          const descendantsVhd = disposableDescendants.value
-          await descendantsVhd.readBlockAllocationTable()
-          debug('got vhd synthetic of descendants')
-          const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
-          debug('got vhd negative')
-
-          // update the stream with the negative vhd stream
-          stream = await negativeVhd.stream()
-          vdis[vdiRef].baseVdi = snapshotCandidate
-        } catch (error) {
-          // can be a broken VHD chain, a vhd chain with a key backup, ....
-          // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
-          warn(`can't use differential restore`, { error })
-          disposableDescendants?.dispose()
-        }
-      }
-      // didn't make a negative stream : fallback to classic stream
-      if (stream === undefined) {
-        debug('use legacy restore')
-        stream = await parentVhd.stream()
-      }
-
-      stream.on('end', disposeOnce)
-      stream.on('close', disposeOnce)
-      stream.on('error', disposeOnce)
-      info('everything is ready, will transfer', stream.length)
-      streams[`${vdiRef}.vhd`] = stream
-    }
-    return {
-      streams,
-      vbds,
-      vdis,
-      version: '1.0.0',
-      vifs,
-      vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
-    }
-  }
-
-  async #decorateIncrementalVmMetadata() {
-    const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
-
-    const ignoredVdis = new Set(
-      Object.entries(mapVdisSrs)
-        .filter(([_, srUuid]) => srUuid === null)
-        .map(([vdiUuid]) => vdiUuid)
-    )
-    let backup
-    if (useDifferentialRestore) {
-      backup = await this._reuseNearestSnapshot(ignoredVdis)
-    } else {
-      backup = await this._adapter.readIncrementalVmBackup(this._metadata, ignoredVdis)
-    }
-    const xapi = this._xapi
-
-    const cache = new Map()
-    const mapVdisSrRefs = {}
-    if (additionnalVmTag !== undefined) {
-      backup.vm.tags.push(additionnalVmTag)
-    }
-    for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
-      mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
-    }
-    const srRef = await resolveUuid(xapi, cache, this._srUuid, 'SR')
-    Object.values(backup.vdis).forEach(vdi => {
-      vdi.SR = mapVdisSrRefs[vdi.uuid] ?? srRef
-    })
-    return backup
-  }
-
   async run() {
     const adapter = this._adapter
     const metadata = this._metadata
     const isFull = metadata.mode === 'full'
 
     const sizeContainer = { size: 0 }
-    const { newMacAddresses } = this._importIncrementalVmSettings
+
     let backup
     if (isFull) {
       backup = await adapter.readFullVmBackup(metadata)
@@ -234,7 +28,12 @@ export class ImportVmBackup {
     } else {
       assert.strictEqual(metadata.mode, 'delta')
 
-      backup = await this.#decorateIncrementalVmMetadata()
+      const ignoredVdis = new Set(
+        Object.entries(this._importIncrementalVmSettings.mapVdisSrs)
+          .filter(([_, srUuid]) => srUuid === null)
+          .map(([vdiUuid]) => vdiUuid)
+      )
+      backup = await adapter.readIncrementalVmBackup(metadata, ignoredVdis)
       Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
@@ -249,7 +48,8 @@ export class ImportVmBackup {
         const vmRef = isFull
           ? await xapi.VM_import(backup, srRef)
           : await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
-              newMacAddresses,
+              ...this._importIncrementalVmSettings,
+              detectBase: false,
             })
 
         await Promise.all([
@@ -259,13 +59,6 @@ export class ImportVmBackup {
             vmRef,
             `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
           ),
-          xapi.call(
-            'VM.set_name_description',
-            vmRef,
-            `Restored on ${formatFilenameDate(+new Date())} from ${adapter._handler._remote.name} -
-             ${metadata.vm.name_description}
-            `
-          ),
         ])
 
         return {
@@ -276,5 +69,3 @@ export class ImportVmBackup {
     )
   }
 }
-
-decorateClass(ImportVmBackup, { _reuseNearestSnapshot: defer })

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -5,7 +5,7 @@ import { createLogger } from '@xen-orchestra/log'
 import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { deduped } from '@vates/disposable/deduped.js'
-import { dirname, join, resolve } from 'node:path'
+import { dirname, join, normalize, resolve } from 'node:path'
 import { execFile } from 'child_process'
 import { mount } from '@vates/fuse-vhd'
 import { readdir, lstat } from 'node:fs/promises'
@@ -18,8 +18,6 @@ import fromEvent from 'promise-toolbox/fromEvent'
 import groupBy from 'lodash/groupBy.js'
 import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
-import reduce from 'lodash/reduce.js'
-import tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'
@@ -36,8 +34,6 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
-const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
-
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -45,23 +41,20 @@ const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 const noop = Function.prototype
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
-const makeRelative = path => resolve('/', path).slice(1)
-const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
 
-async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
-  for (const relativePath of relativePaths) {
-    const realPath = join(realBasePath, relativePath)
-    const virtualPath = join(virtualBasePath, relativePath)
+const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 
-    const stats = await lstat(realPath)
-    const { mode, mtime } = stats
-    const opts = { mode, mtime }
-    if (stats.isDirectory()) {
-      zip.addEmptyDirectory(virtualPath, opts)
-      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
-    } else if (stats.isFile()) {
-      zip.addFile(realPath, virtualPath, opts)
-    }
+async function addDirectory(files, realPath, metadataPath) {
+  const stats = await lstat(realPath)
+  if (stats.isDirectory()) {
+    await asyncMap(await readdir(realPath), file =>
+      addDirectory(files, realPath + '/' + file, metadataPath + '/' + file)
+    )
+  } else if (stats.isFile()) {
+    files.push({
+      realPath,
+      metadataPath,
+    })
   }
 }
 
@@ -189,41 +182,42 @@ export class RemoteAdapter {
     })
   }
 
+  async *_usePartitionFiles(diskId, partitionId, paths) {
+    const path = yield this.getPartition(diskId, partitionId)
+
+    const files = []
+    await asyncMap(paths, file =>
+      addDirectory(files, resolveSubpath(path, file), normalize('./' + file).replace(/\/+$/, ''))
+    )
+
+    return files
+  }
+
   // check if we will be allowed to merge a a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
-    return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
+    return await Disposable.use(openVhd(this.handler, path), vhd => {
       // this baseUuid is not linked with this vhd
       if (!vhd.footer.uuid.equals(packedParentUid)) {
         return false
       }
 
-      // check if all the chain is composed of vhd directory
-      const isVhdDirectory = vhd.checkVhdsClass(VhdDirectory)
+      const isVhdDirectory = vhd instanceof VhdDirectory
       return isVhdDirectory
         ? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
         : !this.useVhdDirectory()
     })
   }
 
-  fetchPartitionFiles(diskId, partitionId, paths, format) {
+  fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
       async function* () {
-        const path = yield this.getPartition(diskId, partitionId)
-        let outputStream
-
-        if (format === 'tgz') {
-          outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
-        } else if (format === 'zip') {
-          const zip = new ZipFile()
-          await addZipEntries(zip, path, '', paths.map(makeRelative))
-          zip.end()
-          ;({ outputStream } = zip)
-        } else {
-          throw new Error('unsupported format ' + format)
-        }
-
+        const files = yield this._usePartitionFiles(diskId, partitionId, paths)
+        const zip = new ZipFile()
+        files.forEach(({ realPath, metadataPath }) => zip.addFile(realPath, metadataPath))
+        zip.end()
+        const { outputStream } = zip
         resolve(outputStream)
         await fromEvent(outputStream, 'end')
       }.bind(this)
@@ -685,13 +679,11 @@ export class RemoteAdapter {
     }
   }
 
-  async outputStream(path, input, { checksum = true, maxStreamLength, streamLength, validator = noop } = {}) {
+  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     const container = watchStreamSize(input)
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
-      maxStreamLength,
-      streamLength,
       async validator() {
         await input.task
         return validator.apply(this, arguments)
@@ -753,37 +745,10 @@ export class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
-    let json
-    let isImmutable = false
-    let remoteIsImmutable = false
-    // if the remote is immutable, check if this metadatas are also immutables
-    try {
-      // this file is not encrypted
-      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
-      remoteIsImmutable = true
-    } catch (error) {
-      if (error.code !== 'ENOENT') {
-        throw error
-      }
-    }
-
-    try {
-      // this will trigger an EPERM error if the file is immutable
-      json = await this.handler.readFile(path, { flag: 'r+' })
-      // s3 handler don't respect flags
-    } catch (err) {
-      // retry without triggerring immutbaility check ,only on immutable remote
-      if (err.code === 'EPERM' && remoteIsImmutable) {
-        isImmutable = true
-        json = await this._handler.readFile(path, { flag: 'r' })
-      } else {
-        throw err
-      }
-    }
     // _filename is a private field used to compute the backup id
     //
     // it's enumerable to make it cacheable
-    const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
+    const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
 
     // backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
     if (typeof metadata.vm.is_a_template === 'number') {
@@ -827,29 +792,6 @@ export class RemoteAdapter {
     }
     return metadata
   }
-
-  #computeTotalBackupSizeRecursively(backups) {
-    return reduce(
-      backups,
-      (prev, backup) => {
-        const _backup = Array.isArray(backup) ? this.#computeTotalBackupSizeRecursively(backup) : backup
-        return {
-          onDisk: prev.onDisk + (_backup.onDisk ?? _backup.size),
-        }
-      },
-      { onDisk: 0 }
-    )
-  }
-
-  async getTotalVmBackupSize() {
-    return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups())
-  }
-
-  async getTotalBackupSize() {
-    const vmBackupSize = await this.getTotalVmBackupSize()
-    // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented by fs
-    return vmBackupSize
-  }
 }
 
 Object.assign(RemoteAdapter.prototype, {
@@ -882,6 +824,8 @@ decorateMethodsWith(RemoteAdapter, {
     debounceResourceFactory,
   ]),
 
+  _usePartitionFiles: Disposable.factory,
+
   getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
 
   getPartition: Disposable.factory,

@xen-orchestra/backups/RestoreMetadataBackup.mjs

@@ -21,12 +21,7 @@ export class RestoreMetadataBackup {
       })
     } else {
       const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
-      const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
-      const data = await handler.readFile(dataFileName)
-
-      // if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
-      const isJson = dataFileName.endsWith('.json')
-      return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
+      return String(await handler.readFile(resolve(backupId, metadata.data ?? 'data.json')))
     }
   }
 }

@xen-orchestra/backups/_backupWorker.mjs

@@ -2,10 +2,7 @@ import { createLogger } from '@xen-orchestra/log'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure'
 
 import Disposable from 'promise-toolbox/Disposable'
-import humanFormat from 'human-format'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
-import mapValues from 'lodash/mapValues.js'
-import ms from 'ms'
 import { compose } from '@vates/compose'
 import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
@@ -23,7 +20,7 @@ createCachedLookup().patchGlobal()
 
 const logger = createLogger('xo:backups:worker')
 catchGlobalErrors(logger)
-const { debug, info, warn } = logger
+const { debug } = logger
 
 class BackupWorker {
   #config
@@ -152,27 +149,6 @@ process.on('message', async message => {
   debug('message received', { message })
 
   if (message.action === 'run') {
-    const resourceStart = process.resourceUsage()
-    const timeStart = process.hrtime.bigint()
-    info('starting backup')
-
-    process.on('exit', exitCode => {
-      const resourceUsage = mapValues(process.resourceUsage(), (end, key) => end - resourceStart[key])
-      const cpuTotal = resourceUsage.userCPUTime + resourceUsage.systemCPUTime
-      const duration = Number((process.hrtime.bigint() - timeStart) / 1000n) // in μs
-
-      info('process will exit', {
-        duration,
-        exitCode,
-        resourceUsage,
-        summary: {
-          duration: ms(duration / 1000),
-          cpuUsage: Math.round((100 * cpuTotal) / duration) + '%',
-          memoryUsage: humanFormat.bytes(resourceUsage.maxRSS * 1024),
-        },
-      })
-    })
-
     const backupWorker = new BackupWorker(message.data)
     try {
       const result = message.runWithLogs
@@ -201,16 +177,8 @@ process.on('message', async message => {
         status: 'failure',
       })
     } finally {
-      info('backup has ended')
       await ignoreErrors.call(backupWorker.debounceResource.flushAll())
       process.disconnect()
-
-      setTimeout(() => {
-        warn('worker process did not exit automatically, forcing...')
-
-        // eslint-disable-next-line n/no-process-exit
-        process.exit()
-      }, 30e3).unref()
     }
   }
 })

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -67,11 +67,6 @@ async function generateVhd(path, opts = {}) {
     await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
   }
 
-  if (opts.blocks) {
-    for (const blockId of opts.blocks) {
-      await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
-    }
-  }
   await vhd.writeBlockAllocationTable()
   await vhd.writeHeader()
   await vhd.writeFooter()
@@ -235,7 +230,7 @@ test('it merges delta of non destroyed chain', async () => {
 
   const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
   // size should be the size of children + grand children after the merge
-  assert.equal(metadata.size, 104960)
+  assert.equal(metadata.size, 209920)
 
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
   // only check deletion
@@ -325,7 +320,6 @@ describe('tests multiple combination ', () => {
         const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
           useAlias,
           mode: vhdMode,
-          blocks: [1, 3],
         })
         const child = await generateVhd(`${basePath}/child.vhd`, {
           useAlias,
@@ -334,7 +328,6 @@ describe('tests multiple combination ', () => {
             parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
             parentUuid: ancestor.footer.uuid,
           },
-          blocks: [1, 2],
         })
         // a grand child  vhd in metadata
         await generateVhd(`${basePath}/grandchild.vhd`, {
@@ -344,7 +337,6 @@ describe('tests multiple combination ', () => {
             parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
             parentUuid: child.footer.uuid,
           },
-          blocks: [2, 3],
         })
 
         // an older parent that was merging in clean
@@ -403,7 +395,7 @@ describe('tests multiple combination ', () => {
 
         const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
         // size should be the size of children + grand children + clean after the merge
-        assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
+        assert.deepEqual(metadata.size, vhdMode === 'file' ? 314880 : undefined)
 
         // broken vhd, non referenced, abandonned should be deleted ( alias and data)
         // ancestor and child should be merged

@xen-orchestra/backups/_cleanVm.mjs

@@ -3,7 +3,7 @@ import sum from 'lodash/sum.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
-import { basename, dirname, resolve } from 'node:path'
+import { dirname, resolve } from 'node:path'
 import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { mergeVhdChain } from 'vhd-lib/merge.js'
@@ -36,32 +36,34 @@ const computeVhdsSize = (handler, vhdPaths) =>
   )
 
 // chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
-  logInfo(`merging VHD chain`, { chain })
+async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
+  if (merge) {
+    logInfo(`merging VHD chain`, { chain })
 
-  let done, total
-  const handle = setInterval(() => {
-    if (done !== undefined) {
-      logInfo('merge in progress', {
-        done,
-        parent: chain[0],
-        progress: Math.round((100 * done) / total),
-        total,
+    let done, total
+    const handle = setInterval(() => {
+      if (done !== undefined) {
+        logInfo('merge in progress', {
+          done,
+          parent: chain[0],
+          progress: Math.round((100 * done) / total),
+          total,
+        })
+      }
+    }, 10e3)
+    try {
+      return await mergeVhdChain(handler, chain, {
+        logInfo,
+        mergeBlockConcurrency,
+        onProgress({ done: d, total: t }) {
+          done = d
+          total = t
+        },
+        removeUnused: remove,
       })
+    } finally {
+      clearInterval(handle)
     }
-  }, 10e3)
-  try {
-    return await mergeVhdChain(handler, chain, {
-      logInfo,
-      mergeBlockConcurrency,
-      onProgress({ done: d, total: t }) {
-        done = d
-        total = t
-      },
-      removeUnused: remove,
-    })
-  } finally {
-    clearInterval(handle)
   }
 }
 
@@ -179,8 +181,7 @@ export async function cleanVm(
   vmDir,
   {
     fixMetadata,
-    remove = false,
-    removeTmp = remove,
+    remove,
     merge,
     mergeBlockConcurrency,
     mergeLimiter = defaultMergeLimiter,
@@ -201,10 +202,6 @@ export async function cleanVm(
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
-    if(removeTmp && basename(path)[0] === '.'){
-      logInfo('deleting temporary VHD', { path })
-        return VhdAbstract.unlink(handler, path)
-    }
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
@@ -442,8 +439,7 @@ export async function cleanVm(
         }
       }
 
-      // no warning because a VHD can be unused for perfectly good reasons,
-      // e.g. the corresponding backup (metadata file) has been deleted
+      logWarn('unused VHD', { path: vhd })
       if (remove) {
         logInfo('deleting unused VHD', { path: vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
@@ -475,20 +471,23 @@ export async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
+      const merged = await limitedMergeVhdChain(handler, chain, {
         logInfo,
         logWarn,
         remove,
+        merge,
         mergeBlockConcurrency,
       })
-      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
-      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
+      if (merged !== undefined) {
+        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
+        metadataWithMergedVhd[metadataPath] = true
+      }
     })
   }
 
   await Promise.all([
     ...unusedVhdsDeletion,
-    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : () => Promise.resolve()),
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
       logWarn('unused XVA', { path })
       if (remove) {
@@ -510,11 +509,12 @@ export async function cleanVm(
 
   // update size for delta metadata with merged VHD
   // check for the other that the size is the same as the real file size
+
   await asyncMap(jsons, async metadataPath => {
     const metadata = backups.get(metadataPath)
 
     let fileSystemSize
-    const mergedSize = metadataWithMergedVhd[metadataPath]
+    const merged = metadataWithMergedVhd[metadataPath] !== undefined
 
     const { mode, size, vhds, xva } = metadata
 
@@ -524,29 +524,26 @@ export async function cleanVm(
         const linkedXva = resolve('/', vmDir, xva)
         try {
           fileSystemSize = await handler.getSize(linkedXva)
-          if (fileSystemSize !== size && fileSystemSize !== undefined) {
-            logWarn('cleanVm: incorrect backup size in metadata', {
-              path: metadataPath,
-              actual: size ?? 'none',
-              expected: fileSystemSize,
-            })
-          }
         } catch (error) {
           // can fail with encrypted remote
         }
       } else if (mode === 'delta') {
+        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+
+        // the size is not computed in some cases (e.g. VhdDirectory)
+        if (fileSystemSize === undefined) {
+          return
+        }
+
         // don't warn if the size has changed after a merge
-        if (mergedSize === undefined) {
-          const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
-          fileSystemSize = await computeVhdsSize(handler, linkedVhds)
-          // the size is not computed in some cases (e.g. VhdDirectory)
-          if (fileSystemSize !== undefined && fileSystemSize !== size) {
-            logWarn('cleanVm: incorrect backup size in metadata', {
-              path: metadataPath,
-              actual: size ?? 'none',
-              expected: fileSystemSize,
-            })
-          }
+        if (!merged && fileSystemSize !== size) {
+          // FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
+          console.warn('cleanVm: incorrect backup size in metadata', {
+            path: metadataPath,
+            actual: size ?? 'none',
+            expected: fileSystemSize,
+          })
         }
       }
     } catch (error) {
@@ -554,19 +551,9 @@ export async function cleanVm(
       return
     }
 
-    // systematically update size and differentials after a merge
-
-    // @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
-    if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
-      metadata.size = mergedSize ?? fileSystemSize ?? size
-
-      if (mergedSize) {
-        // all disks are now key disk
-        metadata.isVhdDifferencing = {}
-        for (const id of Object.values(metadata.vdis ?? {})) {
-          metadata.isVhdDifferencing[`${id}.vhd`] = false
-        }
-      }
+    // systematically update size after a merge
+    if ((merged || fixMetadata) && size !== fileSystemSize) {
+      metadata.size = fileSystemSize
       mustRegenerateCache = true
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })

@xen-orchestra/backups/_incrementalVm.mjs

@@ -1,23 +1,55 @@
+import find from 'lodash/find.js'
 import groupBy from 'lodash/groupBy.js'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import omit from 'lodash/omit.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { CancelToken } from 'promise-toolbox'
 import { compareVersions } from 'compare-versions'
+import { createVhdStreamWithLength } from 'vhd-lib'
 import { defer } from 'golike-defer'
 
 import { cancelableMap } from './_cancelableMap.mjs'
 import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
-import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
+
+export const TAG_BASE_DELTA = 'xo:base_delta'
+
+export const TAG_COPY_SRC = 'xo:copy_of'
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
+const resolveUuid = async (xapi, cache, uuid, type) => {
+  if (uuid == null) {
+    return uuid
+  }
+  let ref = cache.get(uuid)
+  if (ref === undefined) {
+    ref = await xapi.call(`${type}.get_by_uuid`, uuid)
+    cache.set(uuid, ref)
+  }
+  return ref
+}
 
 export async function exportIncrementalVm(
   vm,
-  baseVdis = {},
-  { cancelToken = CancelToken.none, nbdConcurrency = 1, preferNbd } = {}
+  baseVm,
+  {
+    cancelToken = CancelToken.none,
+
+    // Sets of UUIDs of VDIs that must be exported as full.
+    fullVdisRequired = new Set(),
+
+    disableBaseTags = false,
+  } = {}
 ) {
   // refs of VM's VDIs → base's VDIs.
+  const baseVdis = {}
+  baseVm &&
+    baseVm.$VBDs.forEach(vbd => {
+      let vdi, snapshotOf
+      if ((vdi = vbd.$VDI) && (snapshotOf = vdi.$snapshot_of) && !fullVdisRequired.has(snapshotOf.uuid)) {
+        baseVdis[vdi.snapshot_of] = vdi
+      }
+    })
 
   const streams = {}
   const vdis = {}
@@ -38,16 +70,13 @@ export async function exportIncrementalVm(
     }
 
     // Look for a snapshot of this vdi in the base VM.
-    const baseVdi = baseVdis[vdi.$snapshot_of.uuid]
+    const baseVdi = baseVdis[vdi.snapshot_of]
 
     vdis[vdiRef] = {
       ...vdi,
       other_config: {
         ...vdi.other_config,
-        [BASE_DELTA_VDI]: baseVdi?.uuid,
-        [VM_UUID]:
-          vm.$snapshot_of?.uuid ?? // vm is a snapshot
-          vm.uuid, // vm is a not snapshot
+        [TAG_BASE_DELTA]: baseVdi && !disableBaseTags ? baseVdi.uuid : undefined,
       },
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
@@ -57,8 +86,6 @@ export async function exportIncrementalVm(
       baseRef: baseVdi?.$ref,
       cancelToken,
       format: 'vhd',
-      nbdConcurrency,
-      preferNbd,
     })
   })
 
@@ -94,6 +121,13 @@ export async function exportIncrementalVm(
       vifs,
       vm: {
         ...vm,
+        other_config:
+          baseVm && !disableBaseTags
+            ? {
+                ...vm.other_config,
+                [TAG_BASE_DELTA]: baseVm.uuid,
+              }
+            : omit(vm.other_config, TAG_BASE_DELTA),
       },
     },
     'streams',
@@ -109,7 +143,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   $defer,
   incrementalVm,
   sr,
-  { cancelToken = CancelToken.none, newMacAddresses = false } = {}
+  { cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
 ) {
   const { version } = incrementalVm
   if (compareVersions(version, '1.0.0') < 0) {
@@ -119,6 +153,32 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   const vmRecord = incrementalVm.vm
   const xapi = sr.$xapi
 
+  let baseVm
+  if (detectBase) {
+    const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
+    if (remoteBaseVmUuid) {
+      baseVm = find(xapi.objects.all, obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid)
+
+      if (!baseVm) {
+        throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
+      }
+    }
+  }
+
+  const cache = new Map()
+  const mapVdisSrRefs = {}
+  for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
+    mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
+  }
+
+  const baseVdis = {}
+  baseVm &&
+    baseVm.$VBDs.forEach(vbd => {
+      const vdi = vbd.$VDI
+      if (vdi !== undefined) {
+        baseVdis[vbd.VDI] = vbd.$VDI
+      }
+    })
   const vdiRecords = incrementalVm.vdis
 
   // 0. Create suspend_VDI
@@ -130,7 +190,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
         vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
       })
     } else {
-      suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
+      suspendVdi = await xapi.getRecord(
+        'VDI',
+        await xapi.VDI_create({
+          ...vdi,
+          other_config: {
+            ...vdi.other_config,
+            [TAG_BASE_DELTA]: undefined,
+            [TAG_COPY_SRC]: vdi.uuid,
+          },
+          sr: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
+        })
+      )
       $defer.onFailure(() => suspendVdi.$destroy())
     }
   }
@@ -148,6 +219,10 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       ha_always_run: false,
       is_a_template: false,
       name_label: '[Importing…] ' + vmRecord.name_label,
+      other_config: {
+        ...vmRecord.other_config,
+        [TAG_COPY_SRC]: vmRecord.uuid,
+      },
     },
     {
       bios_strings: vmRecord.bios_strings,
@@ -168,11 +243,17 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     const vdi = vdiRecords[vdiRef]
     let newVdi
 
-    if (vdi.baseVdi !== undefined) {
-      newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
+    const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
+    if (remoteBaseVdiUuid) {
+      const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
+      if (!baseVdi) {
+        throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
+      }
+
+      newVdi = await xapi.getRecord('VDI', await baseVdi.$clone())
       $defer.onFailure(() => newVdi.$destroy())
 
-      await newVdi.update_other_config(COPY_OF, vdi.uuid)
+      await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
       if (vdi.virtual_size > newVdi.virtual_size) {
         await newVdi.$callAsync('resize', vdi.virtual_size)
       }
@@ -180,7 +261,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       // suspendVDI has already created
       newVdi = suspendVdi
     } else {
-      newVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
+      newVdi = await xapi.getRecord(
+        'VDI',
+        await xapi.VDI_create({
+          ...vdi,
+          other_config: {
+            ...vdi.other_config,
+            [TAG_BASE_DELTA]: undefined,
+            [TAG_COPY_SRC]: vdi.uuid,
+          },
+          SR: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
+        })
+      )
       $defer.onFailure(() => newVdi.$destroy())
     }
 
@@ -219,16 +311,13 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     // Import VDI contents.
     cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
       for (let stream of ensureArray(streams[`${id}.vhd`])) {
-        if (stream === null) {
-          // we restore a backup and reuse completly a local snapshot
-          continue
-        }
         if (typeof stream === 'function') {
           stream = await stream()
         }
-        await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
+        if (stream.length === undefined) {
+          stream = await createVhdStreamWithLength(stream)
+        }
         await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
-        await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
       }
     }),
 

@xen-orchestra/backups/_otherConfig.mjs

@@ -1,165 +0,0 @@
-import { formatDateTime } from '@xen-orchestra/xapi'
-import assert from 'node:assert/strict'
-// in `other_config` of an incrementally replicated VM or VDI
-// contains the UUID of the object used as a base for an incremental export
-// used to search for the replica of the base before applying a incremental replication
-export const BASE_DELTA_VDI = 'xo:base_delta_vdi'
-
-// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
-//
-// added after the complete replication
-export const REPLICATED_TO_SR_UUID = 'xo:backup:sr'
-
-// in other_config of VDIs of an incrementally replicated VM
-// contains the UUID of the source  exported object (snapshot or VM)
-
-export const COPY_OF = 'xo:copy_of'
-
-export const DATETIME = 'xo:backup:datetime'
-
-export const JOB_ID = 'xo:backup:job'
-export const SCHEDULE_ID = 'xo:backup:schedule'
-
-// contains the number of delta in a chain, stored as a string
-export const DELTA_CHAIN_LENGTH = 'xo:backup:deltaChainLength'
-// contains the string true if this vdi has been exported successfully
-export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
-
-// the VM ( not the snapshot) uuid
-export const VM_UUID = 'xo:backup:vm'
-
-async function listVdiRefs(xapi, vmRef) {
-  return xapi.VM_getDisks(vmRef)
-}
-
-async function applyToVmAndVdis(xapi, vmRef, fn) {
-  const vdiRefs = await listVdiRefs(xapi, vmRef)
-  return Promise.all([fn('VM', vmRef), ...vdiRefs.map(vdiRef => fn('VDI', vdiRef))])
-}
-
-async function getDeltaChainLength(xapi, type, ref) {
-  const otherConfig = await xapi.getField(type, ref, 'other_config')
-  return Number(otherConfig[DELTA_CHAIN_LENGTH] ?? 0)
-}
-
-/**
- * set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @param {Number} length
- * @returns {Promise}
- */
-export async function setVmDeltaChainLength(xapi, vmRef, length) {
-  return applyToVmAndVdis(xapi, vmRef, async (type, ref) => {
-    await xapi.setFieldEntry(type, ref, 'other_config', DELTA_CHAIN_LENGTH, String(length))
-  })
-}
-
-/**
- * Compute the delta chain length of a VM and its associated VDIs
- * if there is a discrependcy, use, the highest value
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export async function getVmDeltaChainLength(xapi, vmRef) {
-  const lengths = await applyToVmAndVdis(xapi, vmRef, async (type, ref) => getDeltaChainLength(xapi, type, ref))
-  return Math.max(...lengths)
-}
-
-/**
- *
- * Reset the other_config field of a VM and its VDIs
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export function resetVmOtherConfig(xapi, vmRef) {
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
-    return xapi.setFieldEntries(type, ref, 'other_config', {
-      [DATETIME]: null,
-      [DELTA_CHAIN_LENGTH]: null,
-      [EXPORTED_SUCCESSFULLY]: null,
-      [JOB_ID]: null,
-      [SCHEDULE_ID]: null,
-      [VM_UUID]: null,
-      // REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
-    })
-  })
-}
-
-/**
- *
- * used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export async function populateVdisOtherConfig(xapi, vmRef) {
-  const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
-  const {
-    [DATETIME]: datetime,
-    [DELTA_CHAIN_LENGTH]: chainLength,
-    [EXPORTED_SUCCESSFULLY]: successfully,
-    [JOB_ID]: jobId,
-    [REPLICATED_TO_SR_UUID]: replicatedTo,
-    [SCHEDULE_ID]: scheduleId,
-    [VM_UUID]: vmUuid,
-  } = otherConfig
-
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
-    xapi.setFieldEntries(type, ref, 'other_config', {
-      [DATETIME]: datetime,
-      [DELTA_CHAIN_LENGTH]: chainLength,
-      [EXPORTED_SUCCESSFULLY]: successfully,
-      [JOB_ID]: jobId,
-      [REPLICATED_TO_SR_UUID]: replicatedTo,
-      [SCHEDULE_ID]: scheduleId,
-      [VM_UUID]: vmUuid,
-    })
-  )
-}
-
-/**
- *
- * set the other_config key related to a backup of a VM and its associated VDIs
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @param {*} settings
- * @returns {PRomise}
- */
-export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, scheduleId, vmUuid, srUuid = null, ...other }) {
-  assert.notEqual(timestamp, undefined)
-  assert.notEqual(jobId, undefined)
-  assert.notEqual(scheduleId, undefined)
-  assert.notEqual(vmUuid, undefined)
-  // srUuid is nullish for backup
-  assert.equal(Object.keys(other).length, 0)
-
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
-    xapi.setFieldEntries(type, ref, 'other_config', {
-      [REPLICATED_TO_SR_UUID]: srUuid,
-      [DATETIME]: formatDateTime(timestamp),
-      [JOB_ID]: jobId,
-      [SCHEDULE_ID]: scheduleId,
-      [VM_UUID]: vmUuid,
-    })
-  )
-}
-/**
- *
- * mark the export of he VM and its VDIs as successfull
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export async function markExportSuccessfull(xapi, vmRef) {
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
-    xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
-  )
-}

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -6,12 +6,11 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
 import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
+import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 
-const noop = Function.prototype
-
 const DEFAULT_REMOTE_VM_SETTINGS = {
   concurrency: 2,
   copyRetention: 0,
@@ -19,10 +18,8 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
   exportRetention: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
-  healthCheckTimeout: '10m',
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
-  nRetriesVmBackupFailures: 0,
   timeout: 0,
   validateVhdStreams: false,
   vmTimeout: 0,
@@ -44,7 +41,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
-
     await Disposable.use(
       () => this._getAdapter(job.sourceRemote),
       () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -66,19 +62,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
-        const queue = new Set(vmsUuids)
-        const taskByVmId = {}
-        const nTriesByVmId = {}
-
         const handleVm = vmUuid => {
-          if (nTriesByVmId[vmUuid] === undefined) {
-            nTriesByVmId[vmUuid] = 0
-          }
-          nTriesByVmId[vmUuid]++
-
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
-          const vmSettings = { ...settings, ...allSettings[vmUuid] }
-          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           const opts = {
             baseSettings,
@@ -87,7 +72,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             healthCheckSr,
             remoteAdapters,
             schedule,
-            settings: vmSettings,
+            settings: { ...settings, ...allSettings[vmUuid] },
             sourceRemoteAdapter,
             throttleStream,
             vmUuid,
@@ -101,46 +86,10 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
           }
 
-          return sourceRemoteAdapter
-            .listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
-            .then(vmBackups => {
-              // avoiding to create tasks for empty directories
-              if (vmBackups.length > 0) {
-                if (taskByVmId[vmUuid] === undefined) {
-                  taskByVmId[vmUuid] = new Task(taskStart)
-                }
-                const task = taskByVmId[vmUuid]
-                return task
-                  .run(async () => {
-                    try {
-                      const result = await vmBackup.run()
-                      task.success(result)
-                      return result
-                    } catch (error) {
-                      if (isLastRun) {
-                        throw error
-                      } else {
-                        Task.warning(`Retry the VM mirror backup due to an error`, {
-                          attempt: nTriesByVmId[vmUuid],
-                          error: error.message,
-                        })
-                        queue.add(vmUuid)
-                      }
-                    }
-                  })
-                  .catch(noop)
-              }
-            })
+          return runTask(taskStart, () => vmBackup.run())
         }
         const { concurrency } = settings
-        const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
-
-        while (queue.size > 0) {
-          const vmIds = Array.from(queue)
-          queue.clear()
-
-          await asyncMapSettled(vmIds, _handleVm)
-        }
+        await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
       }
     )
   }

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -11,11 +11,8 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
 
-const noop = Function.prototype
-
 const DEFAULT_XAPI_VM_SETTINGS = {
   bypassVdiChainsCheck: false,
-  cbtDestroySnapshotData: false,
   checkpointSnapshot: false,
   concurrency: 2,
   copyRetention: 0,
@@ -25,10 +22,8 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   fullInterval: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
-  healthCheckTimeout: '10m',
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
-  nRetriesVmBackupFailures: 0,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,
@@ -58,7 +53,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
-
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -95,98 +89,48 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
-        const queue = new Set(vmIds)
-        const taskByVmId = {}
-        const nTriesByVmId = {}
-
         const handleVm = vmUuid => {
-          const getVmTask = () => {
-            if (taskByVmId[vmUuid] === undefined) {
-              taskByVmId[vmUuid] = new Task(taskStart)
-            }
-            return taskByVmId[vmUuid]
-          }
-          const vmBackupFailed = error => {
-            if (isLastRun) {
-              throw error
-            } else {
-              Task.warning(`Retry the VM backup due to an error`, {
-                attempt: nTriesByVmId[vmUuid],
-                error: error.message,
-              })
-              queue.add(vmUuid)
-            }
-          }
-
-          if (nTriesByVmId[vmUuid] === undefined) {
-            nTriesByVmId[vmUuid] = 0
-          }
-          nTriesByVmId[vmUuid]++
-
-          const vmSettings = { ...settings, ...allSettings[vmUuid] }
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
-          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           return this._getRecord('VM', vmUuid).then(
             disposableVm =>
-              Disposable.use(disposableVm, async vm => {
-                if (taskStart.data.name_label === undefined) {
-                  taskStart.data.name_label = vm.name_label
-                }
-
-                const task = getVmTask()
-                return task
-                  .run(async () => {
-                    const opts = {
-                      baseSettings,
-                      config,
-                      getSnapshotNameLabel,
-                      healthCheckSr,
-                      job,
-                      remoteAdapters,
-                      schedule,
-                      settings: vmSettings,
-                      srs,
-                      throttleStream,
-                      vm,
-                    }
-
-                    let vmBackup
-                    if (job.mode === 'delta') {
-                      vmBackup = new IncrementalXapi(opts)
+              Disposable.use(disposableVm, vm => {
+                taskStart.data.name_label = vm.name_label
+                return runTask(taskStart, () => {
+                  const opts = {
+                    baseSettings,
+                    config,
+                    getSnapshotNameLabel,
+                    healthCheckSr,
+                    job,
+                    remoteAdapters,
+                    schedule,
+                    settings: { ...settings, ...allSettings[vm.uuid] },
+                    srs,
+                    throttleStream,
+                    vm,
+                  }
+                  let vmBackup
+                  if (job.mode === 'delta') {
+                    vmBackup = new IncrementalXapi(opts)
+                  } else {
+                    if (job.mode === 'full') {
+                      vmBackup = new FullXapi(opts)
                     } else {
-                      if (job.mode === 'full') {
-                        vmBackup = new FullXapi(opts)
-                      } else {
-                        throw new Error(`Job mode ${job.mode} not implemented`)
-                      }
+                      throw new Error(`Job mode ${job.mode} not implemented`)
                     }
-
-                    try {
-                      const result = await vmBackup.run()
-                      task.success(result)
-                      return result
-                    } catch (error) {
-                      vmBackupFailed(error)
-                    }
-                  })
-                  .catch(noop) // errors are handled by logs
+                  }
+                  return vmBackup.run()
+                })
               }),
             error =>
-              getVmTask().run(() => {
-                vmBackupFailed(error)
+              runTask(taskStart, () => {
+                throw error
               })
           )
         }
         const { concurrency } = settings
-        const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
-
-        while (queue.size > 0) {
-          const vmIds = Array.from(queue)
-          queue.clear()
-
-          await asyncMapSettled(vmIds, _handleVm)
-        }
+        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
       }
     )
   }

@xen-orchestra/backups/_runners/_PoolMetadataBackup.mjs

@@ -31,7 +31,7 @@ export class PoolMetadataBackup {
     const poolDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${schedule.id}/${pool.$id}`
     const dir = `${poolDir}/${formatFilenameDate(timestamp)}`
 
-    const stream = (await this._exportPoolMetadata()).body
+    const stream = await this._exportPoolMetadata()
     const fileName = `${dir}/data`
 
     const metadata = JSON.stringify(

@xen-orchestra/backups/_runners/_XoMetadataBackup.mjs

@@ -22,13 +22,7 @@ export class XoMetadataBackup {
     const dir = `${scheduleDir}/${formatFilenameDate(timestamp)}`
 
     const data = job.xoMetadata
-    let dataBaseName = './data'
-
-    // JSON data is sent as plain string, binary data is sent as an object with `data` and `encoding properties
-    const isJson = typeof data === 'string'
-    if (isJson) {
-      dataBaseName += '.json'
-    }
+    const dataBaseName = './data.json'
 
     const metadata = JSON.stringify(
       {
@@ -60,7 +54,7 @@ export class XoMetadataBackup {
           async () => {
             const handler = adapter.handler
             const dirMode = this._config.dirMode
-            await handler.outputFile(dataFileName, isJson ? data : Buffer.from(data.data, data.encoding), { dirMode })
+            await handler.outputFile(dataFileName, data, { dirMode })
             await handler.outputFile(metaDataFileName, metadata, {
               dirMode,
             })

@xen-orchestra/backups/_runners/_forkStreamUnpipe.mjs

@@ -29,10 +29,6 @@ export function forkStreamUnpipe(source) {
     if (source.forks === 0) {
       debug('no more forks, destroying original stream')
       source.destroy(new Error('no more consumers for this stream'))
-    } else {
-      // a combination of stream.unpipe, onReadable and onData may stall stream here
-      // force it to flow again since we're piping it
-      source.resume()
     }
   })
   return fork

@xen-orchestra/backups/_runners/_vmRunners/FullRemote.mjs

@@ -1,40 +1,50 @@
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
 import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import { Task } from '../../Task.mjs'
 
 export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return FullRemoteWriter
   }
-
-  _filterTransferList(transferList) {
-    return transferList.filter(this._filterPredicate)
-  }
-
-  async _run() {
+  async _run($defer) {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
-    for (const metadata of transferList) {
-      const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
-      const sizeContainer = watchStreamSize(stream)
+    await this._callWriters(async writer => {
+      await writer.beforeBackup()
+      $defer(async () => {
+        await writer.afterBackup()
+      })
+    }, 'writer.beforeBackup()')
+    if (transferList.length > 0) {
+      for (const metadata of transferList) {
+        const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+        const sizeContainer = watchStreamSize(stream)
 
-      // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
-      await this._callWriters(
-        writer =>
-          writer.run({
-            stream: forkStreamUnpipe(stream),
-            // stream will be forked and transformed, it's not safe to attach additionnal properties to it
-            streamLength: stream.length,
-            timestamp: metadata.timestamp,
-            vm: metadata.vm,
-            vmSnapshot: metadata.vmSnapshot,
-            sizeContainer,
-          }),
-        'writer.run()'
-      )
-      // for healthcheck
-      this._tags = metadata.vm.tags
+        // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
+        await this._callWriters(
+          writer =>
+            writer.run({
+              stream: forkStreamUnpipe(stream),
+              timestamp: metadata.timestamp,
+              vm: metadata.vm,
+              vmSnapshot: metadata.vmSnapshot,
+              sizeContainer,
+            }),
+          'writer.run()'
+        )
+        // for healthcheck
+        this._tags = metadata.vm.tags
+      }
+    } else {
+      Task.info('No new data to upload for this VM')
     }
   }
 }
+
+decorateMethodsWith(FullRemote, {
+  _run: defer,
+})