fix incorrect parentage issue

The deletion of old backups is done globally
For Full backups/replication:
 * retention is computed on the total of the target
 * choose the storage with the most free size ( if possible, since some backup repositories don't have any size used/ free )
 * one vm is written on the same target ( xva, index, checksum )
 * healthcheck ( if enabled ) will run only on the target that received the data
 * the cleanup of old backups/replication is done on the right target ( but considering the full lenght across all the targets )

For incremental: 
 * delta chain are continued on the same target 
 
For mirror : 
if a vm has multiple backups to transfer during the same run, then
they will all go to teh same remote
 
UX : 
* show a toggle on remote/sr selectors
* toggle is disabled if 1 or less targets are selected
* for backup and mirror backup forms
* show the details in the job list
  

More details on the PR page

.eslintrc.js

@@ -105,6 +105,8 @@ module.exports = {
             ],
           },
         ],
+        'no-useless-constructor': 'off',
+        '@typescript-eslint/no-useless-constructor': 'error',
         'no-void': ['error', { allowAsStatement: true }],
         'n/no-missing-import': 'off', // using 'import' plugin instead, to support TS aliases
         'no-redeclare': 'off', // automatically checked by the TypeScript compiler
@@ -169,7 +171,6 @@ module.exports = {
         'vue/no-empty-component-block': 'error',
         'vue/no-multiple-objects-in-class': 'error',
         'vue/no-ref-object-reactivity-loss': 'error',
-        'vue/no-required-prop-with-default': 'error',
         'vue/no-static-inline-styles': 'error',
         'vue/no-template-target-blank': 'error',
         'vue/no-undef-components': ['error', { ignorePatterns: ['RouterLink', 'RouterView', 'I18nT'] }],
@@ -190,18 +191,14 @@ module.exports = {
     // Specific rules for XO i18n-related files
     {
       files: [
-        '@xen-orchestra/lite/**/*.{vue,ts,json}',
-        '@xen-orchestra/web/**/*.{vue,ts,json}',
+        '@xen-orchestra/lite/**/*.{vue,ts}',
+        '@xen-orchestra/web/**/*.{vue,ts}',
         '@xen-orchestra/web-core/**/*.{vue,ts,json}',
       ],
       extends: ['plugin:@intlify/vue-i18n/base'],
       settings: {
         'vue-i18n': {
-          localeDir: [
-            '@xen-orchestra/lite/src/locales/*.json',
-            '@xen-orchestra/web/src/locales/*.json',
-            '@xen-orchestra/web-core/lib/locales/*.json',
-          ],
+          localeDir: ['@xen-orchestra/web-core/lib/locales/*.json'],
           messageSyntaxVersion: '^9.9.0',
         },
       },
@@ -219,13 +216,44 @@ module.exports = {
         '@intlify/vue-i18n/no-raw-text': [
           'error',
           {
-            ignoreText: ['Xen Orchestra Lite', 'Xen Orchestra', 'XO Lite', 'XCP-ng', 'XO 5', '404', '⚠️'],
+            ignoreText: [
+              'Xen Orchestra Lite',
+              'Xen Orchestra',
+              'Xen Orchestra Appliance',
+              'XO Lite',
+              'XCP-ng',
+              'XO 5',
+              '404',
+              '⚠️',
+            ],
           },
         ],
         '@intlify/vue-i18n/no-v-html': 'error',
         '@intlify/vue-i18n/valid-message-syntax': 'error',
         '@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
-        '@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa'] }],
+        '@intlify/vue-i18n/no-missing-keys-in-other-locales': [
+          'error',
+          {
+            ignoreLocales: [
+              'de',
+              'fa',
+              'cs',
+              'es',
+              'sv',
+              'it',
+              'ru',
+              'uk',
+              'nl',
+              'pt-BR',
+              'da',
+              'ko',
+              'nb-NO',
+              'ja',
+              'fi',
+              'pl',
+            ],
+          },
+        ],
       },
     },
     // Specific rules for XO dev-related files
@@ -261,6 +289,8 @@ module.exports = {
     },
   ],
 
+  ignorePatterns: ['@vates/fatfs/'],
+
   parserOptions: {
     ecmaVersion: 13,
     sourceType: 'script',

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -40,8 +40,8 @@ body:
   - type: markdown
     attributes:
       value: |
-        - Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
-        - As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
+        - Don't forget to [read this first](https://docs.xen-orchestra.com/community)
+        - As well as follow [this guide](https://docs.xen-orchestra.com/community#report-a-bug)
   - type: input
     id: xo-sources-commit-number
     attributes:

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continous Integration
+name: Continuous Integration
 on: push
 jobs:
   CI:

.gitignore

@@ -7,6 +7,8 @@
 /@vates/*/node_modules/
 /@xen-orchestra/*/dist/
 /@xen-orchestra/*/node_modules/
+/@xen-orchestra/rest-api/open-api/spec/swagger.json
+/@xen-orchestra/rest-api/src/open-api/routes/routes.ts
 /packages/*/dist/
 /packages/*/node_modules/
 

@vates/async-each/.USAGE.md

@@ -16,7 +16,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 - `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
-- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 
 ```js
 import { asyncEach } from '@vates/async-each'

@vates/async-each/README.md

@@ -34,7 +34,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 - `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
-- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+- `stopOnError`: whether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 
 ```js
 import { asyncEach } from '@vates/async-each'

@vates/async-each/index.js

@@ -11,8 +11,8 @@ class AggregateError extends Error {
 
 /**
  * @template Item
- * @param {Iterable<Item>} iterable
- * @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
+ * @param {Iterable<Item>|AsyncIterable<Item>} iterable
+ * @param {(item: Item, index: number, iterable: Iterable<Item>|AsyncIterable<Item>) => Promise<void>} iteratee
  * @returns {Promise<void>}
  */
 exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
@@ -73,20 +73,20 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
       }
       nextIsRunning = true
       if (running < concurrency) {
-        const cursor = await it.next()
-        if (cursor.done) {
-          next = () => {
-            if (running === 0) {
-              if (errors.length === 0) {
-                resolve()
-              } else {
-                reject(new AggregateError(errors))
+        try {
+          const cursor = await it.next()
+          if (cursor.done) {
+            next = () => {
+              if (running === 0) {
+                if (errors.length === 0) {
+                  resolve()
+                } else {
+                  reject(new AggregateError(errors))
+                }
               }
             }
-          }
-        } else {
-          ++running
-          try {
+          } else {
+            ++running
             const result = iteratee.call(this, cursor.value, index++, iterable)
             let then
             if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
@@ -94,9 +94,9 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
             } else {
               onFulfilled(result)
             }
-          } catch (error) {
-            onRejected(error)
           }
+        } catch (error) {
+          onRejected(error)
         }
         nextIsRunning = false
         return next()

@vates/async-each/index.test.js

@@ -114,6 +114,20 @@ describe('asyncEach', () => {
 
         tracker.verify()
       })
+
+      it('handle error on iterable', async () => {
+        const iteratee = spy(async () => {})
+        const iterable = async function* () {
+          for (const value of [1, 2, 3]) {
+            if (value === 2) {
+              throw new Error('iterator error')
+            }
+            await randomDelay()
+            yield value
+          }
+        }
+        await assert.rejects(() => asyncEach(iterable(), iteratee, { concurrency: 1 }), { message: 'iterator error' })
+      })
     })
   )
 })

@vates/async-each/package.json

@@ -24,7 +24,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "engines": {
     "node": ">=8.10"
   },

@vates/diff/index.js

@@ -7,7 +7,7 @@
  * @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
  * @param {T} data2 - The second data array or buffer to compare.
  * @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
- * @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
+ * @returns {Array<number|T>|undefined} - An array of differences if no callback is provided; otherwise, undefined.
  */
 module.exports = function diff(data1, data2, cb) {
   let result

@vates/disposable/.USAGE.md

@@ -14,7 +14,7 @@ import { deduped } from '@vates/disposable/deduped'
 // the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
 const getConnection = deduped(async function (host)) {
   const connection = new Connection(host)
-  return new Disposabe(connection, () => connection.close())
+  return new Disposable(connection, () => connection.close())
 }, host => [host])
 ```
 

@vates/disposable/README.md

@@ -32,7 +32,7 @@ import { deduped } from '@vates/disposable/deduped'
 // the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
 const getConnection = deduped(async function (host)) {
   const connection = new Connection(host)
-  return new Disposabe(connection, () => connection.close())
+  return new Disposable(connection, () => connection.close())
 }, host => [host])
 ```
 

@vates/fatfs/README.md

@@ -0,0 +1,127 @@
+# @vates/fatfs
+
+This is a fork of the original library [natevw/fatfs](https://github.com/natevw/fatfs).
+
+## Why?
+
+This fork was created to add a missing feature in the original library: the ability to create labels (createLabel).
+The original library did not support this functionality, which was required to make Cloudbase-Init work on windows VMs.
+See: https://github.com/natevw/fatfs/issues/30, https://github.com/natevw/fatfs/pull/31
+
+A standalone FAT16/FAT32 implementation that takes in a block-access interface and exposes something quite similar to `require('fs')` (i.e. the node.js built-in [Filesystem API](http://nodejs.org/api/fs.html)).
+
+## Installation
+
+`npm install fatfs`
+
+## Example
+
+```js
+var fatfs = require('fatfs'),
+  fs = fatfs.createFileSystem(exampleDriver) // see below
+fs.stat('autoexec.bat', function (e, stats) {
+  if (e) console.error(e)
+  else console.log(stats)
+})
+// TODO: open a file and write to it or something…
+```
+
+## API
+
+- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back an instance of an [`fs`](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
+- `'ready'` event — fired on `fs` when initial volume information has been determined and the API is ready to use. It is safe to call other `fs` methods before this fires **only if** you are sure the first sector will be readable and represents a valid FAT volume.
+- `'error'` event — fired if initialization fails for whatever reason.
+
+### Filesystem options
+
+The `opts` dictionary you pass to `fatfs.createFileSystem` can contain any of the following options:
+
+- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overridden to `true`.
+- `noatime` — The FAT filesystem can track the last access time (just a date, actually) but this means every read operation would also incur some write overhead. Defaults to `true`, meaning by default access times will **not** be updated on reads. Set this to `false` to track access times.
+- `modmode` — chooses how `fs.chmod` (and the mode field from `fs.stat`–family calls) should map FAT attributes to POSIX permissions. Set to the number `0111` to map the readonly flag to the user's write bit being unset, and the archive/system/hidden flags to the user/group/other executable bits respectively. Set to the number `07000` to map the readonly flag to _all_ write bits being unset, and the archive/system/hidden flags to the sticky/setgid/setuid bits respectively. Set to `null` for readonly mapping. Defaults to `0111`.
+- `umask` — any bits _set_ in this octal number will be _unset_ in the 'mode' field from `fs.stat`–family calls. It does not affect anything else. Defaults to `process.umask()`, or `0022` if that is unavailable.
+- `uid` — This value will be returned as the 'uid' stat field. It does not affect anything else. Defaults to `process.getuid()`, or `0` if that is unavailable.
+- `gid` — This value will be returned as the 'gid' stat field. It does not affect anything else. Defaults to `process.getgid()`, or `0` if that is unavailable.
+
+(Note that these are similar to the options you could use with a POSIX `mount` operation.)
+
+And that's it! The [rest of the API](http://nodejs.org/api/fs.html) (`fs.readdir`, `fs.open`, `fs.createReadStream`, `fs.appendFile`, etc.) is as documented by the node.js project.
+
+Well, sort of…
+
+## Caveats
+
+### Temporary
+
+- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please, please, please, **make sure you have a backup** of any important drive/image/card you unleash this upon.
+- A few methods are not quite implemented, either: `fs.rename`, `fs.unlink` and `fs.rmdir`, as well as `fs.watchFile`/`fs.unwatchFile` and `fs.watch`. These are Coming Soon™.
+- There are several internal housekeeping items (redundant FAT tables, extra FAT32 information, etc.) that are not done. These do not seem to affect interop, but you may see warnings when repairing a filesystem written by this module.
+- Oh, and not to scare you, but if an IO error happens while writing, the library usually just bails — bubbling an error up to your callback as if it were a hot potato. Although some attempt has been made to do separate writes in the safest order (e.g. allocating an additional file cluster, then appending data into it, and then finally updating the file's size), but this behavior has not been thoroughly audited for all operations. There's certainly no attempt to retry/cleanup/rollback if a multi-step change runs into trouble partway through.
+
+### As-planned
+
+Some of the differences between `fatfs` and the node.js `fs` module are "by design" for architectural simplicity and/or due to underlying FAT limitations.
+
+- There are no `fs.*Sync` methods. (The volume driver is async, not to mention that supporting a separate \*Sync codepath would be an enormous duplication of effort of dubious value.)
+- This module does [almost] no read/write caching. This should be done in your volume driver, but see notes below.
+- You'll need multiple `createFileSystem` instances for multiple volumes; paths are relative to each, and don't share a namespace.
+- The FAT filesystem has no concept of symlinks, and hardlinks are not really an intentional feature. You will get an ENOSYS-like error when trying to create either type of link.
+
+## "Volume driver" API
+
+To use 'fatfs', you must provide a driver object with the following properties/methods:
+
+- `driver.sectorSize` — number of bytes per sector on this device
+- `driver.numSectors` — count of sectors available on this media
+- `driver.readSectors(i, dest, cb)` — Fill `dest` with data starting at the `i`th sector and notify `cb(e)` when finished. You may assume `dest.length` is a multiple of `driver.sectorSize`.
+- `driver.writeSectors(i, data, cb)` — (optional) Write `data` starting at the `i`th sector and notify `cb(e)` when finished. You may assume `data.length` is a multiple of `driver.sectorSize`.
+
+If you do not provide a `writeSectors` method, then `fatfs` will work in readonly mode. Pretty simple, eh? And the 'fatfs' module makes a good effort to check the parameters passed to your driver methods!
+
+**TBD:** to facilitate proper cache handling, this module might add an optional `driver.flush(cb)` method at some point in the future.
+
+Here's an example taken from code used to run this module's own tests:
+
+```js
+// NOTE: this assumes image at `path` has no partition table.
+//       If it did, you'd need to translate positions, natch…
+var fs = require('fs')
+
+exports.createDriverSync = function (path, opts) {
+  opts || (opts = {})
+
+  var secSize = 512,
+    ro = opts.readOnly || false,
+    fd = fs.openSync(path, ro ? 'r' : 'r+'),
+    s = fs.fstatSync(fd)
+
+  return {
+    sectorSize: secSize,
+    numSectors: s.size / secSize,
+    readSectors: function (i, dest, cb) {
+      if (dest.length % secSize) throw Error('Unexpected buffer length!')
+      fs.read(fd, dest, 0, dest.length, i * secSize, function (e, n, d) {
+        cb(e, d)
+      })
+    },
+    writeSectors: ro
+      ? null
+      : function (i, data, cb) {
+          if (data.length % secSize) throw Error('Unexpected buffer length!')
+          fs.write(fd, data, 0, data.length, i * secSize, function (e) {
+            cb(e)
+          })
+        },
+  }
+}
+```
+
+## License
+
+© 2014 Nathan Vander Wilt.
+Funding for this work was provided by Technical Machine, Inc.
+
+Reuse under your choice of:
+
+- [BSD-2-Clause](http://opensource.org/licenses/BSD-2-Clause)
+- [Apache 2.0](http://www.apache.org/licenses/LICENSE-2.0.html)

@vates/fatfs/cache.js

@@ -0,0 +1,60 @@
+var _ = require("./helpers.js");
+
+exports.wrapDriver = function (volume, opts) {
+    opts = _.extend({
+        maxSectors: 2048
+    }, opts);
+    
+    var cache = {},
+        advice = 'NORMAL',
+        secSize = volume.sectorSize;
+    
+    function _freezeBuffer(b) {
+        var f = _.allocBuffer(b.length);
+        b.copy(f);
+        return f;
+    }
+    
+    function addToCache(i, data) {
+        if (advice === 'SEQUENTIAL' || advice === 'NOREUSE') return;
+        data = _freezeBuffer(data);
+        cache[i] = data;
+        //if (data.length > secSize) addToCache(i+1, data.slice(secSize));
+        while (data.length > secSize) {
+            data = data.slice(secSize);
+            cache[++i] = data;
+        }
+        // simple highest-sectors-lose eviction policy for now
+        Object.keys(cache).sort().slice(opts.maxSectors).forEach(function (x) {
+            delete cache[x];
+        });
+        _.log(_.log.DBG, "Cache now contains:", Object.keys(cache).join(','));
+    }
+    
+    return {
+        sectorSize: volume.sectorSize,
+        numSectors: volume.numSectors,
+        advice: function (val) {
+            if (!arguments.length) return advice;
+            else advice = val;
+            if (advice === 'SEQUENTIAL' || advice === 'NOREUSE') cache = {};
+            return this;
+        },
+        readSectors: function (i, dest, cb) {
+            // TODO: handle having partial parts of dest!
+            if (i in cache && dest.length === secSize) {
+                cache[i].copy(dest);
+                setImmediate(cb);
+            } else volume.readSectors(i, dest, function (e) {
+                if (e) cb(e);
+                else addToCache(i, dest), cb();
+            });
+        },
+        writeSectors: (!volume.writeSectors) ? null : function (i, data, cb) {
+            volume.writeSectors(i, data, function (e) {
+                if (e) cb(e);
+                else addToCache(i, data), cb();
+            });
+        }
+    };
+};

@vates/fatfs/chains.js

@@ -0,0 +1,287 @@
+var S = require("./structs.js"),
+    _ = require("./helpers.js");
+
+function _baseChain(vol) {
+    var chain = {};
+    
+    chain.sectorSize = vol._sectorSize;
+    
+    function posFromOffset(off) {
+        var secSize = chain.sectorSize,
+            offset = off % secSize,
+            sector = (off - offset) / secSize;
+        return {sector:sector, offset:offset};
+    }
+    
+    var sectorCache = vol._makeCache();
+    Object.defineProperty(chain, 'cacheAdvice', {
+        enumerable: true,
+        get: function () { return sectorCache.advice(); },
+        set: function (v) { sectorCache.advice(v); }
+    });
+    chain._vol_readSectors = vol._readSectors.bind(vol, sectorCache);
+    chain._vol_writeSectors = vol._writeSectors.bind(vol, sectorCache);
+    
+    // cb(error, bytesRead, buffer)
+    chain.readFromPosition = function (targetPos, buffer, cb) {
+        if (typeof targetPos === 'number') targetPos = posFromOffset(targetPos);
+        if (typeof buffer === 'number') buffer = _.allocBuffer(buffer);
+        /* NOTE: to keep our contract with the volume driver, we need to read on _full_ sector boundaries!
+                 So we divide the read into [up to] three parts: {preface, main, trailer}
+                 This is kind of unfortunate, but in practice should often still be reasonably efficient. */
+        if (targetPos.offset) chain.readSectors(targetPos.sector, _.allocBuffer(chain.sectorSize), function (e,d) {
+            if (e || !d) cb(e, 0, buffer);
+            else {  // copy preface into `buffer`
+                var dBeg = targetPos.offset,
+                    dEnd = dBeg + buffer.length;
+                d.copy(buffer, 0, dBeg, dEnd);
+                if (dEnd > d.length) readMain();
+                else cb(null, buffer.length, buffer);
+            }
+        }); else readMain();
+        function readMain() {
+            var prefaceLen = targetPos.offset && (chain.sectorSize - targetPos.offset),
+                trailerLen = (buffer.length - prefaceLen) % chain.sectorSize,
+                mainSector = (prefaceLen) ? targetPos.sector + 1 : targetPos.sector,
+                mainBuffer = (trailerLen) ? buffer.slice(prefaceLen, -trailerLen) : buffer.slice(prefaceLen);
+            if (mainBuffer.length) chain.readSectors(mainSector, mainBuffer, function (e,d) {
+                if (e || !d) cb(e, prefaceLen, buffer);
+                else if (!trailerLen) cb(null, buffer.length, buffer);
+                else readTrailer();
+            }); else readTrailer();
+            function readTrailer() {
+                var trailerSector = mainSector + (mainBuffer.length / chain.sectorSize);
+                chain.readSectors(trailerSector, _.allocBuffer(chain.sectorSize), function (e,d) {
+                    if (e || !d) cb(e, buffer.length-trailerLen, buffer);
+                    else {
+                        d.copy(buffer, buffer.length-trailerLen, 0, trailerLen);
+                        cb(null, buffer.length, buffer);
+                    }
+                });
+            }
+        }
+    };
+    
+    // cb(error)
+    chain.writeToPosition = function (targetPos, data, cb) {
+        _.log(_.log.DBG, "WRITING", data.length, "bytes at", targetPos, "in", this.toJSON(), data);
+        if (typeof targetPos === 'number') targetPos = posFromOffset(targetPos);
+        
+        var prefaceBuffer = (targetPos.offset) ? data.slice(0, chain.sectorSize-targetPos.offset) : null;
+        if (prefaceBuffer) _modifySector(targetPos.sector, targetPos.offset, prefaceBuffer, function (e) {
+            if (e) cb(e);
+            else if (prefaceBuffer.length < data.length) writeMain();
+            else cb();
+        }); else writeMain();
+        function writeMain() {
+            var prefaceLen = (prefaceBuffer) ? prefaceBuffer.length : 0,
+                trailerLen = (data.length - prefaceLen) % chain.sectorSize,
+                mainSector = (prefaceLen) ? targetPos.sector + 1 : targetPos.sector,
+                mainBuffer = (trailerLen) ? data.slice(prefaceLen, -trailerLen) : data.slice(prefaceLen);
+            if (mainBuffer.length) chain.writeSectors(mainSector, mainBuffer, function (e) {
+                if (e) cb(e);
+                else if (!trailerLen) cb();
+                else writeTrailer();
+            }); else writeTrailer();
+            function writeTrailer() {
+                var trailerSector = mainSector + (mainBuffer.length / chain.sectorSize),
+                    trailerBuffer = data.slice(data.length-trailerLen);     // WORKAROUND: https://github.com/tessel/runtime/issues/721
+                _modifySector(trailerSector, 0, trailerBuffer, cb);
+            }
+        }
+        function _modifySector(sec, off, data, cb) {
+            chain.readSectors(sec, _.allocBuffer(chain.sectorSize), function (e, orig) {
+                if (e) return cb(e);
+                orig || (orig = _.allocBuffer(chain.sectorSize, 0));
+                data.copy(orig, off);
+                chain.writeSectors(sec, orig, cb);
+            });
+        }
+    };
+    
+    return chain;
+};
+
+
+
+exports.clusterChain = function (vol, firstCluster, _parent) {
+    var chain = _baseChain(vol),
+        cache = [firstCluster];
+    
+    chain.firstCluster = firstCluster;
+    
+    function _cacheIsComplete() {
+        return cache[cache.length-1] === 'eof';
+    }
+    
+    function extendCacheToInclude(i, cb) {          // NOTE: may `cb()` before returning!
+        if (i < cache.length) cb(null, cache[i]);
+        else if (_cacheIsComplete()) cb(null, 'eof');
+        else vol.fetchFromFAT(cache[cache.length-1], function (e,d) {
+            if (e) cb(e);
+            else if (typeof d === 'string' && d !== 'eof') cb(S.err.IO());
+            else {
+                cache.push(d);
+                extendCacheToInclude(i, cb);
+            }
+        });
+    }
+    
+    function expandChainToLength(clusterCount, cb) {
+        if (!_cacheIsComplete()) throw Error("Must be called only when cache is complete!");
+        else cache.pop();            // remove 'eof' entry until finished
+        
+        function addCluster(clustersNeeded, lastCluster) {
+            if (!clustersNeeded) cache.push('eof'), cb();
+            else vol.allocateInFAT(lastCluster, function (e, newCluster) {
+                if (e) cb(e);
+                else vol.storeToFAT(lastCluster, newCluster, function (e) {
+                    if (e) return cb(e);
+                    
+                    cache.push(newCluster);
+                    addCluster(clustersNeeded-1, newCluster);
+                });
+            });
+        }
+        addCluster(clusterCount - cache.length, cache[cache.length - 1]);
+    }
+    
+    function shrinkChainToLength(clusterCount, cb) {
+        if (!_cacheIsComplete()) throw Error("Must be called only when cache is complete!");
+        else cache.pop();            // remove 'eof' entry until finished
+        
+        function removeClusters(count, cb) {
+            if (!count) cache.push('eof'), cb();
+            else vol.storeToFAT(cache.pop(), 'free', function (e) {
+                if (e) cb(e);
+                else removeClusters(count - 1, cb);
+            });
+        }
+        // NOTE: for now, we don't remove the firstCluster ourselves; we should though!
+        if (clusterCount) removeClusters(cache.length - clusterCount, cb);
+        else removeClusters(cache.length - 1, cb);
+    }
+    
+    // [{firstSector,numSectors},{firstSector,numSectors},…]
+    function determineSectorGroups(sectorIdx, numSectors, alloc, cb) {
+        var sectorOffset = sectorIdx % vol._sectorsPerCluster,
+            clusterIdx = (sectorIdx - sectorOffset) / vol._sectorsPerCluster,
+            numClusters = Math.ceil((numSectors + sectorOffset) / vol._sectorsPerCluster),
+            chainLength = clusterIdx + numClusters;
+        extendCacheToInclude(chainLength-1, function (e,c) {
+            if (e) cb(e);
+            else if (c === 'eof' && alloc) expandChainToLength(chainLength, function (e) {
+                if (e) cb(e);
+                else _determineSectorGroups();
+            });
+            else _determineSectorGroups();
+        });
+        function _determineSectorGroups() {
+            // …now we have a complete cache
+            var groups = [],
+                _group = null;
+            for (var i = clusterIdx; i < chainLength; ++i) {
+                var c = (i < cache.length) ? cache[i] : 'eof';
+                if (c === 'eof') break;
+                else if (_group && c !== _group._nextCluster) {
+                    groups.push(_group);
+                    _group = null;
+                }
+                if (!_group) _group = {
+                   _nextCluster: c+1,
+                   firstSector: vol._firstSectorOfCluster(c) + sectorOffset,
+                   numSectors: vol._sectorsPerCluster - sectorOffset
+                }; else {
+                    _group._nextCluster += 1;
+                    _group.numSectors += vol._sectorsPerCluster;
+                }
+                sectorOffset = 0;       // only first group is offset
+            }
+            if (_group) groups.push(_group);
+            cb(null, groups, i === chainLength);
+        }
+    }
+    
+    chain.readSectors = function (i, dest, cb) {
+        var groupOffset = 0, groupsPending;
+        determineSectorGroups(i, dest.length / chain.sectorSize, false, function (e, groups, complete) {
+            if (e) cb(e);
+            else if (!complete) groupsPending = -1, _pastEOF(cb);
+            else if ((groupsPending = groups.length)) groups.forEach(function (group) {
+                var groupLength = group.numSectors * chain.sectorSize,
+                    groupBuffer = dest.slice(groupOffset, groupOffset += groupLength);
+                chain._vol_readSectors(group.firstSector, groupBuffer, function (e,d) {
+                    if (e && groupsPending !== -1) groupsPending = -1, cb(e);
+                    else if (--groupsPending === 0) cb(null, dest);
+                });
+            });
+            else cb(null, dest);     // 0-length destination case
+        });
+    };
+    
+    // TODO: does this handle NOSPC condition?
+    chain.writeSectors = function (i, data, cb) {
+        var groupOffset = 0, groupsPending;
+        determineSectorGroups(i, data.length / chain.sectorSize, true, function (e, groups) {
+            if (e) cb(e);
+            else if ((groupsPending = groups.length)) groups.forEach(function (group) {
+                var groupLength = group.numSectors * chain.sectorSize,
+                    groupBuffer = data.slice(groupOffset, groupOffset += groupLength);
+                chain._vol_writeSectors(group.firstSector, groupBuffer, function (e) {
+                    if (e && groupsPending !== -1) groupsPending = -1, cb(e);
+                    else if (--groupsPending === 0) cb();
+                });
+            });
+            else cb();     // 0-length data case
+        });
+    };
+    
+    chain.truncate = function (numSectors, cb) {
+        extendCacheToInclude(Infinity, function (e,c) {
+            if (e) return cb(e);
+            
+            var currentLength = cache.length-1,
+                clustersNeeded = Math.ceil(numSectors / vol._sectorsPerCluster);
+            if (clustersNeeded < currentLength) shrinkChainToLength(clustersNeeded, cb);
+            else if (clustersNeeded > currentLength) expandChainToLength(clustersNeeded, cb);
+            else cb();
+        });
+    };
+    
+    
+    chain.toJSON = function () {
+        return {firstCluster:firstCluster};
+    };
+    
+    return chain;
+};
+
+exports.sectorChain = function (vol, firstSector, numSectors) {
+    var chain = _baseChain(vol);
+    
+    chain.firstSector = firstSector;
+    chain.numSectors = numSectors;
+    
+    chain.readSectors = function (i, dest, cb) {
+        if (i < numSectors) chain._vol_readSectors(firstSector+i, dest, cb);
+        else _pastEOF(cb);
+    };
+    
+    chain.writeSectors = function (i, data, cb) {
+        if (i < numSectors) chain._vol_writeSectors(firstSector+i, data, cb);
+        else _.delayedCall(cb, S.err.NOSPC());
+    };
+    
+    chain.truncate = function (i, cb) {
+        _.delayedCall(cb, S.err.INVAL());
+    };
+    
+    chain.toJSON = function () {
+        return {firstSector:firstSector, numSectors:numSectors};
+    };
+    
+    return chain;
+};
+
+// NOTE: used with mixed feelings, broken out to mark uses
+function _pastEOF(cb) { _.delayedCall(cb, null, null); }

@vates/fatfs/dir.js

@@ -0,0 +1,372 @@
+var S = require("./structs.js"),
+    _ = require("./helpers.js");
+
+var dir = exports;
+
+dir.iterator = function (dirChain, opts) {
+    opts || (opts = {});
+    
+    var cache = {buffer:null, n: null};
+    function getSectorBuffer(n, cb) {
+        if (cache.n === n) cb(null, cache.buffer);
+        else cache.n = cache.buffer = null, dirChain.readSectors(n, _.allocBuffer(dirChain.sectorSize), function (e,d) {
+            if (e) cb(e);
+            else if (!d) return cb(null, null);
+            else {
+                cache.n = n;
+                cache.buffer = d;
+                getSectorBuffer(n, cb);
+            }
+        });
+    }
+    
+    var secIdx = 0,
+        off = {bytes:0},
+        long = null;
+    function getNextEntry(cb) {
+        if (off.bytes >= dirChain.sectorSize) {
+            secIdx += 1;
+            off.bytes -= dirChain.sectorSize;
+        }
+        var entryPos = {chain:dirChain, sector:secIdx, offset:off.bytes};
+        getSectorBuffer(secIdx, function (e, buf) {
+            if (e) return cb(S.err.IO());
+            else if (!buf) return cb(null, null, entryPos);
+            
+            var entryIdx = off.bytes,
+                signalByte = buf[entryIdx];
+            if (signalByte === S.entryDoneFlag) return cb(null, null, entryPos);
+            else if (signalByte === S.entryFreeFlag) {
+                off.bytes += S.dirEntry.size;
+                long = null;
+                if (opts.includeFree) return cb(null, {_free:true,_pos:entryPos}, entryPos);
+                else return getNextEntry(cb);       // usually just skip these
+            }
+            
+            var attrByte = buf[entryIdx+S.dirEntry.fields.Attr.offset],
+                entryType = (attrByte === S.longDirFlag) ? S.longDirEntry : S.dirEntry_simple;
+            var entry = entryType.valueFromBytes(buf, off);
+            entry._pos = entryPos;
+            _.log(_.log.DBG, "entry:", entry, secIdx, entryIdx);
+            if (entryType === S.longDirEntry) {
+                var firstEntry;
+                if (entry.Ord & S.lastLongFlag) {
+                    firstEntry = true;
+                    entry.Ord &= ~S.lastLongFlag;
+                    long = {
+                        name: -1,
+                        sum: entry.Chksum,
+                        _rem: entry.Ord-1,
+                        _arr: []
+                    }
+                }
+                if (firstEntry || long && entry.Chksum === long.sum && entry.Ord === long._rem--) {
+                    var S_lde_f = S.longDirEntry.fields,
+                        namepart = entry.Name1;
+                    if (entry.Name1.length === S_lde_f.Name1.size/2) {
+                        namepart += entry.Name2;
+                        if (entry.Name2.length === S_lde_f.Name2.size/2) {
+                            namepart += entry.Name3;
+                        }
+                    }
+                    long._arr.push(namepart);
+                    if (!long._rem) {
+                        long.name = long._arr.reverse().join('');
+                        delete long._arr;
+                        delete long._rem;
+                    }
+                } else long = null;
+            } else if ((attrByte & 0x08) === 0) {        // NOTE: checks `!entry.Attr.volume_id`
+                var bestName = null;
+                if (long && long.name) {
+                    var pos = entryIdx + S.dirEntry.fields['Name'].offset,
+                        sum = _.checksumName(buf, pos);
+                    if (sum === long.sum) bestName = long.name;
+                }
+                if (!bestName) {
+                    if (signalByte === S.entryIsE5Flag) entry.Name.filename = '\u00E5'+entry.Name.filename.slice(1);
+                    
+                    var nam = entry.Name.filename.replace(/ +$/, ''),
+                        ext = entry.Name.extension.replace(/ +$/, '');
+                    // TODO: lowercase bits http://en.wikipedia.org/wiki/8.3_filename#Compatibility
+                    //       via NTRes, bits 0x08 and 0x10 http://www.fdos.org/kernel/fatplus.txt.1
+                    bestName = (ext) ? nam+'.'+ext : nam;
+                }
+                entry._name = bestName;
+                
+                // OPTIMIZATION: avoid processing any fields for non-matching entries
+                // TODO: we could make this automatic via getters, but…?
+                var _entryBuffer = buf.slice(off.bytes-S.dirEntry.size, off.bytes);
+                entry._full = function () {
+                    var _entry = S.dirEntry.valueFromBytes(_entryBuffer);
+                    _.extend(entry, _entry);
+                    entry._size = entry.FileSize;
+                    entry._firstCluster = (entry.FstClusHI << 16) + entry.FstClusLO;
+                    return entry;
+                };
+                
+                long = null;
+                return cb(null, entry, entryPos);
+            } else long = null;
+            getNextEntry(cb);
+        });
+    }
+    
+    function iter(cb) {
+        getNextEntry(cb);
+        return iter;            // TODO: previous value can't be re-used, so why make caller re-assign?
+    }
+    return iter;
+};
+
+function _updateEntry(vol, entry, newStats) {
+    if ('size' in newStats) entry._size = entry.FileSize = newStats.size;
+    
+    if ('_touch' in newStats) newStats.archive = newStats.atime = newStats.mtime = true;
+    if ('archive' in newStats) entry.Attr.archive = true;           // TODO: also via newStats.mode?
+    
+    var _now;
+    function applyDate(d, prefix, timeToo, tenthToo) {
+        if (d === true) d = _now || (_now = new Date());
+        entry[prefix+'Date'] = {year:d.getFullYear()-1980, month:d.getMonth()+1, day:d.getDate()};
+        if (timeToo) {
+            entry[prefix+'Time'] = {hours:d.getHours(), minutes:d.getMinutes(), seconds_2:d.getSeconds()>>>1};
+            if (tenthToo) {
+                var msec = (d.getSeconds() % 2)*1000 + d.getMilliseconds();
+                entry[prefix+'TimeTenth'] = Math.floor(msec / 100);
+            }
+        }
+    }
+    if ('ctime' in newStats) applyDate(newStats.ctime, 'Crt', true, true);
+    if ('mtime' in newStats) applyDate(newStats.mtime, 'Wrt', true);
+    if ('atime' in newStats) applyDate(newStats.atime, 'LstAcc');
+    
+    if ('mode' in newStats) {
+        entry.Attr.directory = (newStats.mode & S._I.FDIR) ? true : false;
+        entry.Attr.volume_id = (newStats.mode & S._I.FREG) ? false : true;
+        if (vol.opts.modmode === 0111) {
+            entry.Attr.archive = (newStats.mode & S._I.XUSR) ? true : false;
+            entry.Attr.system  = (newStats.mode & S._I.XGRP) ? true : false;
+            entry.Attr.hidden  = (newStats.mode & S._I.XOTH) ? true : false;
+            entry.Attr.readonly = (newStats.mode & S._I.WUSR) ? false : true;
+        } else if (vol.opts.modmode === 07000) {
+            entry.Attr.archive = (newStats.mode & S._I.SVTX) ? true : false;
+            entry.Attr.system  = (newStats.mode & S._I.SGID) ? true : false;
+            entry.Attr.hidden  = (newStats.mode & S._I.SUID) ? true : false;
+            entry.Attr.readonly = (
+                newStats.mode & S._I.WUSR ||
+                newStats.mode & S._I.WGRP ||
+                newStats.mode & S._I.WOTH
+            ) ? false : true;
+        }
+    }
+    
+    if ('firstCluster' in newStats) {
+        entry.FstClusLO = newStats.firstCluster & 0xFFFF;
+        entry.FstClusHI = newStats.firstCluster >>> 16;
+        entry._firstCluster = newStats.firstCluster;
+    }
+    return entry;
+}
+
+dir.makeStat = function (vol, entry) {
+    var stats = {};     // TODO: return an actual `instanceof fs.Stat` somehow?
+    
+    stats.isFile = function () {
+        return (!entry.Attr.volume_id && !entry.Attr.directory);
+    };
+    stats.isDirectory = function () {
+        return entry.Attr.directory;
+    };
+    stats.isBlockDevice = function () { return false; }
+    stats.isCharacterDevice = function () { return false; }
+    stats.isSymbolicLink = function () { return false; }
+    stats.isFIFO = function () { return false; }
+    stats.isSocket = function () { return false; }
+    stats.size = entry.FileSize;
+    stats.blksize = vol._sectorsPerCluster*vol._sectorSize;
+    stats.blocks = Math.ceil(stats.size / stats.blksize) || 1;
+    stats.nlink = 1;
+    
+    stats.mode = S._I.RUSR | S._I.RGRP | S._I.ROTH;
+    if (!entry.Attr.readonly) stats.mode |= S._I.WUSR | S._I.WGRP | S._I.WOTH;
+    if (entry.Attr.directory) stats.mode |= S._I.FDIR;
+    else if (!entry.Attr.volume_id) stats.mode |= S._I.FREG;
+    // NOTE: discussion at https://github.com/natevw/fatfs/issues/7
+    if (vol.opts.modmode === 0111) {
+        // expose using executable bits, like Samba
+        if (entry.Attr.archive) stats.mode |= S._I.XUSR;
+        if (entry.Attr.system)  stats.mode |= S._I.XGRP;
+        if (entry.Attr.hidden)  stats.mode |= S._I.XOTH;
+    } else if (vol.opts.modmode === 07000) {
+        // expose using setXid/sticky bits, like MKS
+        if (entry.Attr.archive) stats.mode |= S._I.SVTX;
+        if (entry.Attr.system)  stats.mode |= S._I.SGID;
+        if (entry.Attr.hidden)  stats.mode |= S._I.SUID;
+    }
+    
+    stats.mode &= ~vol.opts.umask;
+    stats.uid =  vol.opts.uid;
+    stats.gid = vol.opts.gid;
+    
+    function extractDate(prefix) {
+        var date = entry[prefix+'Date'],
+            time = entry[prefix+'Time'] || {hours:0, minutes:0, seconds_2:0},
+            secs = time.seconds_2 * 2,
+            sect = entry[prefix+'TimeTenth'] || 0;
+        if (sect > 100) {
+            secs += 1;
+            sect -= 100;
+        }
+        return new Date(date.year+1980, date.month-1, date.day, time.hours, time.minutes, secs, sect*100);
+    }
+    stats.atime = extractDate('LstAcc');
+    stats.mtime = extractDate('Wrt');
+    stats.ctime = extractDate('Crt');
+    
+    entry = {           // keep immutable copy (with only the fields we need)
+        Attr: _.extend({},entry.Attr),
+    };
+    
+    return stats;
+};
+
+dir.init = function (vol, dirInfo, cb) {
+    var dirChain = dirInfo.chain,
+        isRootDir = ('numSectors' in dirChain),    // HACK: all others would be a clusterChain
+        initialCluster = _.allocBuffer(dirChain.sectorSize*vol._sectorsPerCluster),
+        entriesOffset = {bytes:0};
+    initialCluster.fill(0);
+    function writeEntry(name, clusterNum) {
+        while (name.length < 8) name += " ";
+        S.dirEntry.bytesFromValue(_updateEntry(vol, {
+            Name: {filename:name, extension:"   "},
+            Attr: {directory:true}
+        }, {firstCluster:clusterNum, _touch:true,ctime:true}), initialCluster, entriesOffset);
+    }
+    if (!isRootDir) {
+        writeEntry(".", dirChain.firstCluster);
+        writeEntry("..", dirInfo.parent.chain.firstCluster);
+    };
+    dirChain.writeToPosition(0, initialCluster, cb);
+};
+
+dir.addFile = function (vol, dirChain, entryInfo, opts, cb) {
+    if (typeof opts === 'function') {
+        cb = opts;
+        opts = {};
+    }
+    var name = entryInfo.name,
+        entries = [], mainEntry;
+    entries.push(mainEntry = {
+        Name: _.shortname(name),
+        Attr: {directory:opts.dir||false},
+        _name: name
+    });
+    if (1 || mainEntry.Name._lossy) {         // HACK: always write long names until `._lossy` is more useful!
+        var workaroundTessel427 = ('\uFFFF'.length !== 1);
+        if (workaroundTessel427) throw Error("Your JS runtime does not have proper Unicode string support. (If Tessel, is your firmware up-to-date?)");
+        
+        // name entries should be 0x0000-terminated and 0xFFFF-filled
+        var S_lde_f = S.longDirEntry.fields,
+            ENTRY_CHUNK_LEN = (S_lde_f.Name1.size + S_lde_f.Name2.size + S_lde_f.Name3.size)/2,
+            partialLen = name.length % ENTRY_CHUNK_LEN,
+            paddingNeeded = partialLen && (ENTRY_CHUNK_LEN - partialLen);
+        if (paddingNeeded--) name += '\u0000';
+        while (paddingNeeded-- > 0) name += '\uFFFF';
+        // now fill in as many entries as it takes
+        var off = 0,
+            ord = 1;
+        while (off < name.length) entries.push({
+            Ord: ord++,
+            Name1: name.slice(off, off+=S_lde_f.Name1.size/2),
+            Attr_raw: S.longDirFlag,
+            Chksum: null,
+            Name2: name.slice(off, off+=S_lde_f.Name2.size/2),
+            Name3: name.slice(off, off+=S_lde_f.Name3.size/2)
+        });
+        entries[entries.length - 1].Ord |= S.lastLongFlag;
+    }
+    
+    if (entryInfo.tail) {
+        var name = mainEntry.Name.filename,
+            suffix = '~'+entryInfo.tail,
+            endIdx = name.indexOf(' '),
+            sufIdx = (~endIdx) ? Math.min(endIdx, name.length-suffix.length) : name.length-suffix.length;
+        if (sufIdx < 0) return cb(S.err.NAMETOOLONG());         // TODO: would EXIST be more correct?
+        mainEntry.Name.filename = name.slice(0,sufIdx)+suffix+name.slice(sufIdx+suffix.length);
+        _.log(_.log.DBG, "Shortname amended to:", mainEntry.Name);
+    }
+    
+    vol.allocateInFAT(dirChain.firstCluster || 2, function (e,fileCluster) {
+        if (e) return cb(e);
+        
+        var nameBuf = S.dirEntry.fields['Name'].bytesFromValue(mainEntry.Name),
+            nameSum = _.checksumName(nameBuf);
+        // TODO: finalize initial properties… (via `opts.mode` instead?)
+        _updateEntry(vol, mainEntry, {firstCluster:fileCluster, size:0, ctime:true,_touch:true});
+        mainEntry._pos = _.adjustedPos(vol, entryInfo.target, S.dirEntry.size*(entries.length-1));
+        entries.slice(1).forEach(function (entry) {
+            entry.Chksum = nameSum;
+        });
+        entries.reverse();
+        if (entryInfo.lastEntry) entries.push({});
+        
+        var entriesData = _.allocBuffer(S.dirEntry.size*entries.length),
+            dataOffset = {bytes:0};
+        entries.forEach(function (entry) {
+            var entryType = ('Ord' in entry) ? S.longDirEntry : S.dirEntry;
+            entryType.bytesFromValue(entry, entriesData, dataOffset);
+        });
+        
+        _.log(_.log.DBG, "Writing", entriesData.length, "byte directory entry", mainEntry, "into", dirChain.toJSON(), "at", entryInfo.target);
+        dirChain.writeToPosition(entryInfo.target, entriesData, function (e) {
+            // TODO: if we get error, what/should we clean up?
+            if (e) cb(e);
+            else cb(null, mainEntry, vol.chainForCluster(fileCluster, dirChain));
+        });
+    });
+};
+
+dir.findInDirectory = function (vol, dirChain, name, opts, cb) {
+    var matchName = name.toUpperCase(),
+        tailName = (opts.prepareForCreate) ? _.shortname(name) : null,
+        maxTail = 0;
+    
+    function processNext(next) {
+        next = next(function (e, d, entryPos) {
+            if (e) cb(e);
+            else if (!d) cb(S.err.NOENT(), {tail:maxTail, target:entryPos, lastEntry:true});
+            else if (d._free) processNext(next);         // TODO: look for long enough reusable run
+            else if (d._name.toUpperCase() === matchName) return cb(null, d._full());
+            else if (!opts.prepareForCreate) processNext(next);
+            else {
+                var dNum = 1,
+                    dName = d.Name.filename,
+                    dTail = dName.match(/(.*)~(\d+)/);
+                if (dTail) {
+                    dNum = +dTail[2];
+                    dName = dTail[1];
+                }
+                if (tailName.extension === d.Name.extension &&
+                    tailName.filename.indexOf(dName) === 0)
+                {
+                    maxTail = Math.max(dNum+1, maxTail);
+                }
+                processNext(next);
+            }
+        });
+    }
+    processNext(dir.iterator(dirChain, {includeFree:(0 && opts.prepareForCreate)}));
+};
+
+dir.updateEntry = function (vol, entry, newStats, cb) {
+    if (!entry._pos || !entry._pos.chain) throw Error("Entry source unknown!");
+    
+    var entryPos = entry._pos,
+        newEntry = _updateEntry(vol, entry, newStats),
+        data = S.dirEntry.bytesFromValue(newEntry);
+    _.log(_.log.DBG, "UPDATING ENTRY", newStats, newEntry, entryPos, data);
+    // TODO: if write fails, then entry becomes corrupt!
+    entryPos.chain.writeToPosition(entryPos, data, cb);
+};

@vates/fatfs/helpers.js

@@ -0,0 +1,206 @@
+var S = require("./structs.js"),
+    _xok = require('xok');
+
+// ponyfills for older node.js
+exports.allocBuffer = Buffer.alloc || function (len, val) {
+    var b = Buffer(len);
+    if (arguments.length > 1) b.fill(val);
+    return b;
+};
+exports.bufferFrom = Buffer.from || function (arg0, arg1) {
+    return (arguments.length > 1) ? Buffer(arg0, arg1) : Buffer(arg0);
+};
+
+// flag for WORKAROUND: https://github.com/tessel/beta/issues/380
+exports.workaroundTessel380 = !Buffer.from && function () {
+    var b = Buffer([0]),
+        s = b.slice(0);
+    return ((s[0] = 0xFF) !== b[0]);
+}();
+
+
+// WORKAROUND: https://github.com/tessel/beta/issues/433
+var oldslice;
+if (!Buffer.alloc && Buffer(5).slice(10).length < 0) oldslice = Buffer.prototype.slice, Buffer.prototype.slice = function (s, e) {
+    if (s > this.length) s = this.length;
+    // ~WORKAROUND: https://github.com/tessel/beta/issues/434
+    return (arguments.length > 1) ? oldslice.call(this, s, e) : oldslice.call(this, s);
+}
+
+exports.absoluteSteps = function (path) {
+    var steps = [];
+    path.split('/').forEach(function (str) {
+        // NOTE: these should actually be fine, just wasteful…
+        if (str === '..') steps.pop();
+        else if (str && str !== '.') steps.push(str);
+    });
+    return steps.map(exports.longname);
+};
+
+exports.absolutePath = function (path) {
+    return '/'+exports.absoluteSteps(path).join('/');
+};
+
+exports.parseFlags = function (flags) {
+    // read, write, append, create, truncate, exclusive
+    var info, _dir;           // NOTE: there might be more clever ways to "parse", but…
+    if (flags[0] === '\\') {
+        // internal flag used internally to `fs.open` directories without `S.err.ISDIR()`
+        flags = flags.slice(1);
+        _dir = true;
+    }
+    switch (flags) {
+        case 'r':   info = {read:true, write:false, create:false}; break;
+        case 'r+':  info = {read:true, write:true, create:false}; break;
+        case 'rs':  info = {read:true, write:false, create:false, sync:true}; break;
+        case 'rs+': info = {read:true, write:true, create:false, sync:true}; break;
+        case 'w':   info = {read:false, write:true, create:true, truncate:true}; break;
+        case 'wx':  info = {read:false, write:true, create:true, exclusive:true}; break;
+        case 'w+':  info = {read:true, write:true, create:true, truncate:true}; break;
+        case 'wx+': info = {read:true, write:true, create:true, exclusive:true}; break;
+        case 'a':   info = {read:false, write:true, create:true, append:true}; break;
+        case 'ax':  info = {read:false, write:true, create:true, append:true, exclusive:true}; break;
+        case 'a+':  info = {read:true, write:true, create:true, append:true}; break;
+        case 'ax+': info = {read:true, write:true, create:true, append:true, exclusive:true}; break;
+        default: throw Error("Unknown mode: "+flags);       // TODO: throw as `S.err.INVAL`
+    }
+    if (info.sync) throw Error("Mode not implemented.");    // TODO: what would this require of us?
+    if (_dir) info._openDir = true;
+    return info;
+};
+
+
+// TODO: these are great candidates for special test coverage!
+var _snInvalid = /[^A-Z0-9$%'-_@~`!(){}^#&.]/g;         // NOTE: '.' is not valid but we split it away
+exports.shortname = function (name) {
+    var lossy = false;
+    // TODO: support preservation of case for otherwise non-lossy name!
+    name = name.toUpperCase().replace(/ /g, '').replace(/^\.+/, '');
+    name = name.replace(_snInvalid, function () {
+        lossy = true;
+        return '_';
+    });
+    
+    var parts = name.split('.'),
+        basis3 = parts.pop(),
+        basis8 = parts.join('');
+    if (!parts.length) {
+        basis8 = basis3;
+        basis3 = '   ';
+    }
+    if (basis8.length > 8) {
+        basis8 = basis8.slice(0,8);
+        // NOTE: technically, spec's "lossy conversion" flag is NOT set by excess length.
+        //       But since lossy conversion and truncated names both need a numeric tail…
+        lossy = true;
+    } else while (basis8.length < 8) basis8 += ' ';
+    if (basis3.length > 3) {
+        basis3 = basis3.slice(0,3);
+        lossy = true;
+    } else while (basis3.length < 3) basis3 += ' ';
+    
+    return {filename:basis8, extension:basis3, _lossy:lossy};
+    return {basis:[basis8,basis3], lossy:lossy};
+};
+//shortname("autoexec.bat") => {basis:['AUTOEXEC','BAT'],lossy:false}
+//shortname("autoexecutable.batch") => {basis:['AUTOEXEC','BAT'],lossy:true}
+// TODO: OS X stores `shortname("._.Trashes")` as ['~1', 'TRA'] — should we?
+
+var _lnInvalid = /[^a-zA-Z0-9$%'-_@~`!(){}^#&.+,;=[\] ]/g;
+exports.longname = function (name) {
+    name = name.trim().replace(/\.+$/, '').replace(_lnInvalid, function (c) {
+        if (c.length > 1) throw Error("Internal problem: unexpected match length!");
+        if (c.charCodeAt(0) > 127) return c;
+        else throw Error("Invalid character "+JSON.stringify(c)+" in name.");
+        lossy = true;
+        return '_';
+    });
+    if (name.length > 255) throw Error("Name is too long.");
+    return name;
+};
+
+function nameChkSum(sum, c) {
+    return ((sum & 1) ? 0x80 : 0) + (sum >>> 1) + c & 0xFF;
+}
+
+// WORKAROUND: https://github.com/tessel/beta/issues/335
+function reduceBuffer(buf, start, end, fn, res) {
+    // NOTE: does not handle missing `res` like Array.prototype.reduce would
+    for (var i = start; i < end; ++i) {
+        res = fn(res, buf[i]);
+    }
+    return res;
+}
+
+exports.checksumName = function (buf,off) {
+    off || (off = 0);
+    var len = S.dirEntry.fields['Name'].size;
+    return reduceBuffer(buf, off, off+len, nameChkSum, 0);
+};
+
+
+/* comparing C rounding trick from FAT spec with Math.ceil
+function tryBoth(d) {
+    var a = ((D.RootEntCnt * 32) + (D.BytsPerSec - 1)) / D.BytsPerSec >>> 0,
+        b = Math.ceil((D.RootEntCnt * 32) / D.BytsPerSec);
+    if (b !== a) console.log("try", b, a, (b === a) ? '' : '*');
+    return (b === a);
+}
+// BytsPerSec — "may take on only the following values: 512, 1024, 2048 or 4096"
+[512, 1024, 2048, 4096].forEach(function (bps) {
+    // RootEntCnt — "a count that when multiplied by 32 results in an even multiple of BPB_BytsPerSec"
+    for (var evenMultiplier = 0; evenMultiplier < 1024*1024*16; evenMultiplier += 2) {
+        var rec = (bps * evenMultiplier) / 32;
+        tryBoth({RootEntCnt:rec, BytsPerSec:bps});
+    }
+});
+*/
+
+exports.fmtHex = function (n, ff) {
+    return (1+ff+n).toString(16).slice(1);
+};
+
+exports.delayedCall = function (fn) {
+    if (!fn) throw Error("No function provided!");      // debug aid
+    var ctx = this,
+        args = Array.prototype.slice.call(arguments, 1);
+    setImmediate(function () {
+        fn.apply(ctx, args);
+    });
+};
+
+exports.adjustedPos = function (vol, pos, bytes) {
+    var _pos = {
+        chain: pos.chain,
+        sector: pos.sector,
+        offset: pos.offset + bytes
+    }, secSize = vol._sectorSize;
+    while (_pos.offset >= secSize) {
+        _pos.sector += 1;
+        _pos.offset -= secSize;
+    }
+    return _pos;
+};
+
+exports.extend = _xok;
+
+var _prevDbg = Date.now(),
+    _thresh = 50;
+
+function log(level) {
+    if (level < log.level) return;
+    
+    var now = Date.now(),
+        diff = now - _prevDbg;
+    arguments[0] = ((diff < _thresh) ? " " : '') + diff.toFixed(0)  + "ms";
+    console.log.apply(console, arguments);
+    _prevDbg = now;
+}
+log.DBG = -4;
+log.INFO = -3;
+log.WARN = -2;
+log.ERR = -1;
+
+log.level = log.WARN;
+
+exports.log = log;

@vates/fatfs/img_volume.js

@@ -0,0 +1,25 @@
+var fs = require('fs');
+
+exports.createDriverSync = function (path, opts) {
+    opts || (opts = {});
+    
+    var secSize = 512,
+        ro = opts.readOnly || false,
+        fd = fs.openSync(path, (ro) ? 'r' : 'r+'),
+        s = fs.fstatSync(fd);
+    
+    return {
+        sectorSize: secSize,
+        numSectors: s.size / secSize,
+        readSectors: function (i, dest, cb) {
+            fs.read(fd, dest, 0, dest.length, i*secSize, function (e,n,d) {
+                cb(e,d);
+            });
+        },
+        writeSectors: (ro) ? null : function (i, data, cb) {
+            fs.write(fd, data, 0, data.length, i*secSize, function (e) {
+                cb(e);
+            });
+        }
+    };
+};

@vates/fatfs/index.js

@@ -0,0 +1,771 @@
+var events = require('events'),
+  streams = require('stream'),
+  fifolock = require('fifolock'),
+  S = require('./structs.js'),
+  _ = require('./helpers.js')
+
+//_.log.level = _.log.DBG;
+
+exports.createFileSystem = function (volume, opts, cb) {
+  if (typeof opts === 'function') {
+    cb = opts
+    opts = null
+  }
+  opts = _.extend(
+    {
+      // c.f. https://www.kernel.org/doc/Documentation/filesystems/vfat.txt
+      ro: false,
+      noatime: true,
+      modmode: 0111, // or `07000`
+      umask: 'umask' in process ? process.umask() : 0022,
+      uid: 'getuid' in process ? process.getuid() : 0,
+      gid: 'getgid' in process ? process.getgid() : 0,
+    },
+    opts
+  )
+  if (!volume.writeSectors) opts.ro = true
+  if (opts.ro) opts.noatime = true // natch
+
+  var fs = new events.EventEmitter(),
+    vol = null,
+    dir = require('./dir.js'),
+    c = require('./chains.js'),
+    q = fifolock()
+
+  var GROUP =
+    _.log.level < _.log.INFO
+      ? q.TRANSACTION_WRAPPER.bind({
+          postAcquire: function (proceed) {
+            _.log(_.log.DBG, '=== Starting GROUP ===')
+            proceed()
+          },
+          preRelease: function (finish) {
+            _.log(_.log.DBG, '=== Finishing GROUP ===')
+            finish()
+          },
+        })
+      : q.TRANSACTION_WRAPPER
+
+  q.acquire(function (unlock) {
+    // because of this, callers can start before 'ready'
+    var d = _.allocBuffer(volume.sectorSize)
+    volume.readSectors(0, d, function (e) {
+      if (e) fs.emit('error', e)
+      else {
+        try {
+          init(d)
+        } catch (e) {
+          fs.emit('error', e)
+          unlock()
+          return
+        }
+        fs.emit('ready')
+      }
+      unlock()
+    })
+  })
+
+  if (cb) fs.on('error', cb).on('ready', cb.bind(null, null))
+
+  function init(bootSector) {
+    vol = require('./vol.js').init(volume, opts, bootSector)
+    fs._dirIterator = dir.iterator.bind(dir)
+
+    var entryInfoByPath = {},
+      baseEntry = {
+        _refs: 0,
+        _record: function () {
+          entryInfoByPath[this.path] = this
+          if (this.parent) this.parent.retain()
+          return this
+        },
+        retain: function () {
+          if (!this._refs) this._record()
+          this._refs += 1
+          return this
+        },
+        release: function () {
+          this._refs -= 1
+          if (!this._refs) this._rescind()
+        },
+        _rescind: function () {
+          if (this.parent) this.parent.release()
+          delete entryInfoByPath[this.path]
+        },
+      }
+    fs._createSharedEntry = function (path, entry, chain, parent) {
+      return _.extend(Object.create(baseEntry), {
+        _refs: 0, // WORKAROUND: https://github.com/tessel/beta/issues/455
+        path: path,
+        entry: entry,
+        chain: chain,
+        parent: parent,
+      }).retain()
+    }
+    fs._createSharedEntry('/', { Attr: { directory: true } }, vol.rootDirectoryChain)
+    fs._sharedEntryForSteps = function (steps, opts, cb) {
+      // NOTE: may `cb` before returning!
+      var path = steps.join('/') || '/',
+        name = steps.pop(), // n.b.
+        info = entryInfoByPath[path]
+      if (info) cb(null, info.retain())
+      else
+        fs._sharedEntryForSteps(steps, {}, function (e, parentInfo) {
+          // n.b. `steps` don't include `name`
+          if (e) cb(e)
+          else if (!parentInfo.entry.Attr.directory) cb(S.err.NOTDIR())
+          else
+            dir.findInDirectory(vol, parentInfo.chain, name, opts, function (e, entry) {
+              if (e && !opts.prepareForCreate) cb(e)
+              else if (e) cb(e, { missingChild: _.extend(entry, { name: name }), parent: parentInfo })
+              else cb(null, fs._createSharedEntry(path, entry, vol.chainForCluster(entry._firstCluster), parentInfo))
+            })
+        })
+    }
+
+    fs._updateEntry = dir.updateEntry.bind(dir, vol)
+    fs._makeStat = dir.makeStat.bind(dir, vol)
+    fs._addFile = dir.addFile.bind(dir, vol)
+    fs._initDir = dir.init.bind(dir, vol)
+  }
+
+  /**** ---- CORE API ---- ****/
+
+  // NOTE: we really don't share namespace, but avoid first three anyway…
+  var fileDescriptors = [null, null, null]
+
+  fs.open = function (path, flags, mode, cb, _n_) {
+    if (typeof mode === 'function') {
+      _n_ = cb
+      cb = mode
+      mode = 0666
+    }
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = { flags: null, entry: null, chain: null, pos: 0 },
+          f = _.parseFlags(flags)
+        if (vol.opts.ro && (f.write || f.create || f.truncate)) return _.delayedCall(cb, S.err.ROFS())
+        else _fd.flags = f
+
+        fs._sharedEntryForSteps(_.absoluteSteps(path), { prepareForCreate: f.create }, function (e, info) {
+          if (e && !(e.code === 'NOENT' && f.create && info)) cb(e)
+          else if (e)
+            fs._addFile(info.parent.chain, info.missingChild, { dir: f._openDir }, function (e, newEntry, newChain) {
+              if (e) cb(e)
+              else finish(fs._createSharedEntry(_.absolutePath(path), newEntry, newChain, info.parent))
+            })
+          else if (info && f.exclusive) cb(S.err.EXIST())
+          else if (info.entry.Attr.directory && !f._openDir) cb(S.err.ISDIR())
+          else if (f.write && info.entry.Attr.readonly) cb(S.err.ACCES())
+          else finish(info)
+          function finish(fileInfo) {
+            var fd = fileDescriptors.push(_fd) - 1
+            _fd.info = fileInfo
+            _fd.entry = fileInfo.entry
+            _fd.chain = fileInfo.chain
+            if (f.append) _fd.pos = _fd.entry._size
+            if (f._openDir) _fd.chain.cacheAdvice = 'WILLNEED'
+            if (f.truncate && _fd.entry._size)
+              fs.ftruncate(
+                fd,
+                0,
+                function (e) {
+                  cb(e, fd)
+                },
+                '_nested_'
+              )
+            else _.delayedCall(cb, null, fd) // (delay in case fs._sharedEntryForSteps all cached!)
+          }
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.fstat = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) _.delayedCall(cb, S.err.BADF())
+        else _.delayedCall(cb, null, fs._makeStat(_fd.entry))
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.futimes = function (fd, atime, mtime, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        // NOTE: ctime would get touched on POSIX; but we map that to create time!
+        else fs._updateEntry(_fd.entry, { atime: atime || true, mtime: mtime || true }, cb)
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.fchmod = function (fd, mode, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else {
+          mode &= S._I._chmoddable
+          if (_fd.entry.Attr.directory) mode |= S._I.FDIR
+          else if (!_fd.entry.Attr.volume_id) mode |= S._I.FREG
+          fs._updateEntry(_fd.entry, { mode: mode }, cb)
+        }
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.read = function (fd, buf, off, len, pos, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) return _.delayedCall(cb, S.err.BADF())
+
+        var _pos = pos === null ? _fd.pos : pos,
+          _len = Math.min(len, _fd.entry._size - _pos),
+          _buf = buf.slice(off, off + _len)
+        _fd.chain.readFromPosition(_pos, _buf, function (e, bytes, slice) {
+          if (_.workaroundTessel380) _buf.copy(buf, off) // WORKAROUND: https://github.com/tessel/beta/issues/380
+          _fd.pos = _pos + bytes
+          if (e || vol.opts.noatime) finish(e)
+          else fs._updateEntry(_fd.entry, { atime: true }, finish)
+          function finish(e) {
+            cb(e, bytes, buf)
+          }
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs._readdir = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) return _.delayedCall(cb, S.err.BADF())
+
+        var entryNames = [],
+          getNextEntry = fs._dirIterator(_fd.chain)
+        function processNext() {
+          getNextEntry(function (e, d) {
+            if (e) cb(e)
+            else if (!d && !entryNames.length)
+              cb(null, entryNames) // WORKAROUND: https://github.com/tessel/beta/issues/435
+            else if (!d)
+              cb(null, entryNames.sort()) // NOTE: sort not required, but… [simplifies tests for starters!]
+            else {
+              if (d._name !== '.' && d._name !== '..') entryNames.push(d._name)
+              processNext()
+            }
+          })
+        }
+        processNext()
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs._mkdir = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else fs._initDir(_fd.info, cb)
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.write = function (fd, buf, off, len, pos, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) return _.delayedCall(cb, S.err.BADF())
+
+        var _pos = pos === null || _fd.flags.append ? _fd.pos : pos,
+          _buf = buf.slice(off, off + len)
+        if (_pos > _fd.entry._size) {
+          // TODO: handle huge jumps by zeroing clusters individually?
+          var padLen = _pos - _fd.entry._size,
+            padBuf = _.allocBuffer(padLen + _buf.length)
+          padBuf.fill(0x00, 0, padLen)
+          _buf.copy(padBuf, padLen)
+          _pos = _fd.entry._size
+          _buf = padBuf
+        }
+        _fd.chain.writeToPosition(_pos, _buf, function (e) {
+          _fd.pos = _pos + len
+          var newSize = Math.max(_fd.entry._size, _fd.pos),
+            newInfo = { size: newSize, _touch: true }
+          fs._updateEntry(_fd.entry, newInfo, function (ee) {
+            cb(e || ee, len, buf)
+          })
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.ftruncate = function (fd, len, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) return _.delayedCall(cb, S.err.BADF())
+
+        var newStats = { size: len, _touch: true }
+        // NOTE: we order operations for best state in case of only partial success
+        if (len === _fd.entry._size) _.delayedCall(cb)
+        else if (len < _fd.entry._size)
+          fs._updateEntry(_fd.entry, newStats, function (e) {
+            if (e) cb(e)
+            else _fd.chain.truncate(Math.ceil(len / _fd.chain.sectorSize), cb)
+          })
+        // TODO: handle huge file expansions without as much memory pressure
+        else
+          _fd.chain.writeToPosition(_fd.entry._size, _.allocBuffer(len - _fd.entry._size, 0x00), function (e) {
+            if (e) cb(e)
+            else fs._updateEntry(_fd.entry, newStats, cb)
+          })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  // 'NORMAL', 'SEQUENTIAL', 'RANDOM', 'WILLNEED', 'DONTNEED', 'NOREUSE'
+  fs._fadviseSync = function (fd, off, len, advice) {
+    if (off !== 0 || len !== 0) throw Error('Cache advise can currently be given only for whole file!')
+    var _fd = fileDescriptors[fd]
+    if (!_fd) throw S.err.BADF()
+    else _fd.chain.cacheAdvice = advice
+  }
+
+  fs.fsync = function (fd, cb) {
+    // NOTE: we'll need to flush write cache here once we have one…
+    var _fd = fileDescriptors[fd]
+    if (!_fd) _.delayedCall(cb, S.err.BADF())
+    else _.delayedCall(cb)
+  }
+
+  fs.close = function (fd, cb) {
+    var _fd = fileDescriptors[fd]
+    if (!_fd) _.delayedCall(cb, S.err.BADF())
+    else setTimeout(_fd.info.release.bind(_fd.info), 500), _.delayedCall(cb, (fileDescriptors[fd] = null))
+  }
+
+  /* STREAM WRAPPERS */
+
+  var workaroundTessel436
+  try {
+    new require('stream').Readable({ encoding: 'utf8' })
+    new streams.Readable({ encoding: 'utf8' })
+  } catch (e) {
+    workaroundTessel436 = true
+  }
+
+  function _createStream(StreamType, path, opts) {
+    // [NOT REALLY A] WORKAROUND: https://github.com/tessel/beta/issues/436
+    if (workaroundTessel436 && 'encoding' in opts) {
+      console.warn('Tessel does not currently support encoding option for Readable streams, discarding!')
+      delete opts.encoding
+    }
+
+    var fd = opts.fd !== null ? opts.fd : '_opening_',
+      pos = opts.start,
+      stream = new StreamType(opts)
+
+    if (fd === '_opening_')
+      fs.open(path, opts.flags, opts.mode, function (e, _fd) {
+        if (e) {
+          fd = '_open_error_'
+          stream.emit('error', e)
+        } else {
+          fd = _fd
+          fs._fadviseSync(fd, 0, 0, 'SEQUENTIAL')
+          stream.emit('open', fd)
+        }
+      })
+
+    function autoClose(tombstone) {
+      // NOTE: assumes caller will clear `fd`
+      if (opts.autoClose)
+        fs.close(fd, function (e) {
+          if (e) stream.emit('error', e)
+          else stream.emit('close')
+        })
+      fd = tombstone
+    }
+
+    if (StreamType === streams.Readable) {
+      stream._read = function (n) {
+        var buf
+        // TODO: optimize to fetch at least a full sector regardless of `n`…
+        n = Math.min(n, opts.end - pos)
+        if (fd === '_opening_')
+          stream.once('open', function () {
+            stream._read(n)
+          })
+        else if (pos > opts.end) stream.push(null)
+        else if (n > 0)
+          (buf = _.allocBuffer(n)),
+            fs.read(fd, buf, 0, n, pos, function (e, n, d) {
+              if (e) {
+                autoClose('_read_error_')
+                stream.emit('error', e)
+              } else stream.push(n ? d.slice(0, n) : null)
+            }),
+            (pos += n)
+        else stream.push(null)
+      }
+
+      stream.once('end', function () {
+        autoClose('_ended_')
+      })
+    } else if (StreamType === streams.Writable) {
+      stream.bytesWritten = 0
+
+      stream._write = function (data, _enc, cb) {
+        if (fd === '_opening_')
+          stream.once('open', function () {
+            stream._write(data, null, cb)
+          })
+        else
+          fs.write(fd, data, 0, data.length, pos, function (e, n) {
+            if (e) {
+              autoClose('_write_error_')
+              cb(e)
+            } else {
+              stream.bytesWritten += n
+              cb()
+            }
+          }),
+            (pos += data.length)
+      }
+
+      stream.once('finish', function () {
+        autoClose('_finished_')
+      })
+    }
+
+    return stream
+  }
+
+  fs.createReadStream = function (path, opts) {
+    return _createStream(
+      streams.Readable,
+      path,
+      _.extend(
+        {
+          start: 0,
+          end: Infinity,
+          flags: 'r',
+          mode: 0666,
+          encoding: null,
+          fd: null, // ??? see https://github.com/joyent/node/issues/7708
+          autoClose: true,
+        },
+        opts
+      )
+    )
+  }
+
+  fs.createWriteStream = function (path, opts) {
+    return _createStream(
+      streams.Writable,
+      path,
+      _.extend(
+        {
+          start: 0,
+          flags: 'w',
+          mode: 0666,
+          //encoding: null,   // see https://github.com/joyent/node/issues/7710
+          fd: null, // ??? see https://github.com/joyent/node/issues/7708
+          autoClose: true,
+        },
+        opts,
+        { decodeStrings: true, objectMode: false }
+      )
+    )
+  }
+
+  /* PATH WRAPPERS (albeit the only public interface for some folder operations) */
+
+  function _fdOperation(path, opts, fn, cb) {
+    cb = GROUP(cb, function () {
+      opts.advice || (opts.advice = 'NORMAL')
+      fs.open(
+        path,
+        opts.flag,
+        function (e, fd) {
+          if (e) cb(e)
+          else
+            fs._fadviseSync(fd, 0, 0, opts.advice),
+              fn(fd, function () {
+                var ctx = this,
+                  args = arguments
+                fs.close(
+                  fd,
+                  function (closeErr) {
+                    cb.apply(ctx, args)
+                  },
+                  '_nested_'
+                )
+              })
+        },
+        '_nested_'
+      )
+    })
+  }
+
+  fs.stat = fs.lstat = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r' },
+      function (fd, cb) {
+        fs.fstat(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.exists = function (path, cb) {
+    fs.stat(path, function (err) {
+      cb(err ? false : true)
+    })
+  }
+
+  fs.readFile = function (path, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'r')
+    opts.advice || (opts.advice = 'NOREUSE')
+    _fdOperation(
+      path,
+      opts,
+      function (fd, cb) {
+        fs.fstat(
+          fd,
+          function (e, stat) {
+            if (e) return cb(e)
+            else {
+              var buffer = _.allocBuffer(stat.size)
+              fs.read(
+                fd,
+                buffer,
+                0,
+                buffer.length,
+                null,
+                function (e) {
+                  if (e) cb(e)
+                  else cb(null, opts.encoding ? buffer.toString(opts.encoding) : buffer)
+                },
+                '_nested_'
+              )
+            }
+          },
+          '_nested_'
+        )
+      },
+      cb
+    )
+  }
+
+  fs.writeFile = function (path, data, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'w')
+    opts.advice || (opts.advice = 'NOREUSE')
+    _fdOperation(
+      path,
+      opts,
+      function (fd, cb) {
+        if (typeof data === 'string') data = _.bufferFrom(data, opts.encoding || 'utf8')
+        fs.write(
+          fd,
+          data,
+          0,
+          data.length,
+          null,
+          function (e) {
+            cb(e)
+          },
+          '_nested_'
+        )
+      },
+      cb
+    )
+  }
+
+  fs.appendFile = function (path, data, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'a')
+    fs.writeFile(path, data, opts, cb)
+  }
+
+  fs.truncate = function (path, len, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r+' },
+      function (fd, cb) {
+        fs.ftruncate(fd, len, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.readdir = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r' },
+      function (fd, cb) {
+        fs._readdir(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.mkdir = function (path, mode, cb) {
+    if (typeof mode === 'function') {
+      cb = mode
+      mode = 0777
+    }
+    _fdOperation(
+      path,
+      { flag: '\\wx' },
+      function (fd, cb) {
+        fs._mkdir(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.utimes = function (path, atime, mtime, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r+' },
+      function (fd, cb) {
+        fs.futimes(fd, atime, mtime, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.chmod = fs.lchmod = function (path, mode, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r+' },
+      function (fd, cb) {
+        fs.fchmod(fd, mode, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.chown = fs.lchown = function (path, uid, gid, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r+' },
+      function (fd, cb) {
+        fs.fchown(fd, uid, gid, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  /* STUBS */
+
+  fs.link = function (src, dst, cb) {
+    // NOTE: theoretically we _could_ do hard links [with untracked `stat.nlink` count…]
+    _.delayedCall(cb, S.err.NOSYS())
+  }
+
+  fs.symlink = function (src, dst, type, cb) {
+    if (typeof type === 'function') {
+      cb = type
+      type = null
+    }
+    _.delayedCall(cb, S.err.NOSYS())
+  }
+
+  fs.readlink = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r' },
+      function (fd, cb) {
+        // the named file is *never* a symbolic link…
+        // NOTE: we still use _fdOperation for catching e.g. NOENT/NOTDIR errors…
+        cb(S.err.INVAL())
+      },
+      cb
+    )
+  }
+
+  fs.realpath = function (path, cache, cb) {
+    if (typeof cache === 'function') {
+      cb = cache
+      cache = null
+    }
+    if (cache)
+      _.delayedCall(cb, S.err.NOSYS()) // TODO: what would be involved here?
+    else
+      _fdOperation(
+        path,
+        { flag: '\\r' },
+        function (fd, cb) {
+          cb(null, _.absolutePath(path))
+        },
+        cb
+      )
+  }
+
+  fs.fchown = function (fd, uid, gid, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else _.delayedCall(cb, S.err.NOSYS())
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  // See https://github.com/natevw/fatfs/pull/31/files
+  fs.createLabel = function (name, cb) {
+    fs.open(name, 'a', (e, _fd) => {
+      if (e) {
+        cb(e)
+      } else {
+        let fd = fileDescriptors[_fd]
+        fd.entry.Attr.volume_id = true
+        fd.entry.FstClusLO = 0
+        fd.entry.FstClusHI = 0
+
+        fs._updateEntry(fd.entry, {}, cb)
+      }
+    })
+  }
+
+  return fs
+}

@vates/fatfs/make_sample.sh

@@ -0,0 +1,36 @@
+#! /bin/sh
+
+
+# NOTE: this only works on OS X
+# TODO: Linux could use mkfs stuff without attach?
+
+# ./make_sample.sh /tmp/fat16.img FAT16
+
+FILE=$1
+: ${FILE:?must be provided as first argument}
+
+TYPE=$2
+: ${TYPE:?must be provided as second argument}
+
+SIZE=$3                             # in MB
+case "$TYPE" in
+    "FAT12") : ${SIZE:=1}           # <128
+    ;;
+    "FAT16") : ${SIZE:=5}           # >4
+    ;;
+    "FAT32") : ${SIZE:=33}          # >32
+    ;;
+    *) : ${SIZE:=1}                 # e.g. ExFAT
+esac
+
+echo Making $TYPE of $SIZE MB at $FILE
+
+if [[ "$TYPE" = FAT* ]]
+then
+    TYPE="MS-DOS $TYPE"
+fi
+
+dd if=/dev/zero of=$FILE bs=1048576 count=$SIZE
+DEV=`hdiutil attach $FILE -nomount`
+diskutil eraseVolume "$TYPE" "FATFS TEST" $DEV
+hdiutil detach $DEV

@vates/fatfs/package.json

@@ -0,0 +1,34 @@
+{
+  "private": false,
+  "name": "@vates/fatfs",
+  "version": "0.11.1",
+  "description": "fs implementation on top of raw FAT16/FAT32 block source",
+  "main": "index.js",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "repository": {
+    "directory": "@vates/fatfs",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "keywords": [
+    "filesystem",
+    "fs",
+    "fat",
+    "fat32",
+    "fat16"
+  ],
+  "author": "Vates SAS",
+  "license": "Apache-2.0",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fatfs",
+  "dependencies": {
+    "fifolock": "^1.0.0",
+    "struct-fu": "^1.2.1",
+    "xok": "^1.0.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

@vates/fatfs/structs.js

@@ -0,0 +1,255 @@
+// see http://staff.washington.edu/dittrich/misc/fatgen103.pdf
+// and http://www.cse.scu.edu/~tschwarz/COEN252_09/Lectures/FAT.html
+
+var _ = require('struct-fu'),
+    __ = require("./helpers.js");
+
+var bootBase = _.struct([
+    _.byte('jmpBoot', 3),
+    _.char('OEMName', 8),
+    _.uint16le('BytsPerSec'),
+    _.uint8('SecPerClus'),
+    _.uint16le('ResvdSecCnt'),      // Rsvd in table, but Resvd in calcs…
+    _.uint8('NumFATs'),
+    _.uint16le('RootEntCnt'),
+    _.uint16le('TotSec16'),
+    _.uint8('Media'),
+    _.uint16le('FATSz16'),
+    _.uint16le('SecPerTrk'),
+    _.uint16le('NumHeads'),
+    _.uint32le('HiddSec'),
+    _.uint32le('TotSec32')
+]);
+
+var bootInfo = _.struct([
+    _.uint8('DrvNum'),
+    _.uint8('Reserved1'),
+    _.uint8('BootSig'),
+    _.uint32le('VolID'),
+    _.char('VolLab', 11),
+    _.char('FilSysType', 8)
+]);
+
+exports.boot16 = _.struct([
+    bootBase,
+    bootInfo
+]);
+
+exports.boot32 = _.struct([
+    bootBase,
+    _.uint32le('FATSz32'),
+    _.struct('ExtFlags', [
+        _.ubit('NumActiveFAT', 4),
+        _.ubit('_reserved1', 3),
+        _.bool('MirroredFAT'),
+        _.ubit('_reserved2', 8)
+    ].reverse()),
+    _.struct('FSVer', [
+        _.uint8('Major'),
+        _.uint8('Minor')
+    ]),
+    _.uint32le('RootClus'),
+    _.uint16le('FSInfo'),
+    _.uint16le('BkBootSec'),
+    _.byte('Reserved', 12),
+    bootInfo
+]);
+
+
+var _time = _.struct([
+    _.ubit('hours',5),
+    _.ubit('minutes',6),
+    _.ubit('seconds_2',5)
+]), time = {
+    valueFromBytes: function (buf, off) {
+        off || (off = {bytes:0});
+        
+        var _buf = __.bufferFrom([buf[off.bytes+1], buf[off.bytes+0]]),
+            val = _time.valueFromBytes(_buf);
+        off.bytes += this.size;
+        return val;
+    },
+    bytesFromValue: function (val, buf, off) {
+        val || (val = {hours:0, minutes:0, seconds_2:0});
+        buf || (buf = __.allocBuffer(this.size));
+        off || (off = {bytes:0});
+        
+        var _buf = _time.bytesFromValue(val);
+        buf[off.bytes+1] = _buf[0];
+        buf[off.bytes+0] = _buf[1];
+        off.bytes += this.size;
+        return buf;
+    },
+    size: _time.size
+};
+
+var _date = _.struct([
+    _.ubit('year',7),
+    _.ubit('month',4),
+    _.ubit('day',5)
+]), date = {
+    valueFromBytes: function (buf, off) {
+        off || (off = {bytes:0});
+        
+        var _buf = __.bufferFrom([buf[off.bytes+1], buf[off.bytes+0]]),
+            val = _date.valueFromBytes(_buf);
+        off.bytes += this.size;
+        return val;
+    },
+    bytesFromValue: function (val, buf, off) {
+        val || (val = {year:0, month:0, day:0});
+        buf || (buf = __.allocBuffer(this.size));
+        off || (off = {bytes:0});
+        
+        var _buf = _date.bytesFromValue(val);
+        buf[off.bytes+1] = _buf[0];
+        buf[off.bytes+0] = _buf[1];
+        off.bytes += this.size;
+        return buf;
+    },
+    size: _date.size
+};
+
+exports.dirEntry = _.struct([
+    _.struct('Name', [
+        _.char('filename',8),
+        _.char('extension',3)
+    ]),
+    _.struct('Attr', [
+        _.bool('readonly'),
+        _.bool('hidden'),
+        _.bool('system'),
+        _.bool('volume_id'),
+        _.bool('directory'),
+        _.bool('archive'),
+        _.ubit('reserved', 2)
+    ].reverse()),
+    _.byte('NTRes', 1),
+    _.uint8('CrtTimeTenth'),
+    _.struct('CrtTime', [time]),
+    _.struct('CrtDate', [date]),
+    _.struct('LstAccDate', [date]),
+    _.uint16le('FstClusHI'),
+    _.struct('WrtTime', [time]),
+    _.struct('WrtDate', [date]),
+    _.uint16le('FstClusLO'),
+    _.uint32le('FileSize')
+]);
+exports.entryDoneFlag = 0x00;
+exports.entryFreeFlag = 0xE5;
+exports.entryIsE5Flag = 0x05;
+
+exports.dirEntry_simple = _.struct([
+    _.struct('Name', [
+        _.char('filename',8),
+        _.char('extension',3)
+    ]),
+    _.padTo(exports.dirEntry.size)
+    /*
+    _.uint8('Attr_raw'),
+    _.byte('NTRes', 1),
+    _.byte('Crt_raw', 1+2+2),
+    _.byte('Lst_raw', 2),
+    _.uint16le('FstClusHI'),
+    _.byte('Wrt_raw', 2+2),
+    _.uint16le('FstClusLO'),
+    _.uint32le('FileSize')
+    */
+]);
+
+
+exports.lastLongFlag = 0x40;
+exports.longDirFlag = 0x0F;
+exports.longDirEntry = _.struct([
+    _.uint8('Ord'),
+    _.char16le('Name1', 10),
+    _.uint8('Attr_raw'),
+    _.uint8('Type'),
+    _.uint8('Chksum'),
+    _.char16le('Name2', 12),
+    _.uint16le('FstClusLO'),
+    _.char16le('Name3', 4)
+]);
+
+if (exports.longDirEntry.size !== exports.dirEntry.size) throw Error("Structs ain't right!");
+
+exports.fatField = {
+    'fat12': _.struct('Status', [
+        _.ubit('field0bc', 8),
+        _.ubit('field1c', 4),
+        _.ubit('field0a', 4),
+        _.ubit('field1ab', 8),
+    ]),
+    'fat16': _.uint16le('Status'),
+    'fat32': _.uint32le('Status')       // more properly this 4 bits reserved + uint28le
+};
+
+exports.fatPrefix = {
+    'fat12': 0xF00,
+    'fat16': 0xFF00,
+    'fat32': 0x0FFFFF00
+};
+
+exports.fatStat = {
+    free: 0x00,
+    _undef: 0x01,
+    rsvMin: 0xF0,
+    bad: 0xF7,
+    eofMin: 0xF8,
+    eof: 0xFF
+};
+
+exports._I = {
+    RUSR: 0400,
+    WUSR: 0200,
+    XUSR: 0100,
+    
+    RGRP: 0040,
+    WGRP: 0020,
+    XGRP: 0010,
+    
+    ROTH: 0004,
+    WOTH: 0002,
+    XOTH: 0001,
+    
+    SUID: 04000,
+    SGID: 02000,
+    SVTX: 01000,
+    
+    FDIR: 040000,
+    FREG: 0100000,
+};
+
+exports._I.RWXU = exports._I.RUSR | exports._I.WUSR | exports._I.XUSR;
+exports._I.RWXG = exports._I.RGRP | exports._I.WGRP | exports._I.XGRP;
+exports._I.RWXO = exports._I.ROTH | exports._I.WOTH | exports._I.XOTH;
+exports._I._sss = exports._I.SUID | exports._I.SGID | exports._I.SVTX;
+exports._I._chmoddable = exports._I.RWXU | exports._I.RWXG | exports._I.RWXO | exports._I._sss;
+
+var _errors = {
+    IO: "Input/output error",
+    NOENT: "No such file or directory",
+    INVAL: "Invalid argument",
+    EXIST: "File exists",
+    NAMETOOLONG: "Filename too long",
+    NOSPC: "No space left on device",
+    NOSYS: "Function not supported",
+    ROFS: "ROFLCopter file system",
+    NOTDIR: "Not a directory",
+    BADF: "Bad file descriptor",
+    EXIST: "File exists",
+    ISDIR: "Is a directory",
+    ACCES: "Permission denied",
+    NOSYS: "Function not implemented",
+    _TODO: "Not implemented yet!"
+};
+
+exports.err = {};
+Object.keys(_errors).forEach(function (sym) {
+    var msg = _errors[sym];
+    exports.err[sym] = function () {
+        var e = new Error(msg);
+        e.code = sym;
+        return e;
+    };
+});

@vates/fatfs/test.js

@@ -0,0 +1,338 @@
+// can be called from CLI with image type, absolute path to an image, or, required as a module
+
+var _ = require("./helpers.js");
+
+var type = process.argv[2];
+if (module.parent) exports.startTests = startTests;
+else if (!type) throw "Usage: node test [FAT12|FAT16|FAT32|ExFAT|…|/path/to/image]";
+else if (type[0] === '/') testWithImage(type);
+else {
+    var uniq = Math.random().toString(36).slice(2),
+        IMG = require('os').tmpdir()+"fatfs-test-"+uniq+".img";
+    require('child_process').exec("./make_sample.sh "+JSON.stringify(IMG)+" "+JSON.stringify(type), function (e,out,err) {
+        if (e) throw e;
+        console.warn(err.toString());
+        //console.log(out.toString());
+        testWithImage(IMG);
+//console.log("open", IMG);
+//return;
+        require('fs').unlink(IMG, function (e) {
+            if (e) console.warn("Error cleaning up test image", e);
+        });
+    });
+}
+
+function testWithImage(imagePath) {
+    var vol = require("./img_volume.js").createDriverSync(imagePath);
+    startTests(vol);
+}
+
+
+function startTests(vol, waitTime) {
+    var fatfs = require("./"),
+        fs = fatfs.createFileSystem(vol, {umask:0020, uid:99, gid:42});
+    
+    waitTime || (waitTime = 0.5e3);
+    
+    [
+        'mkdir','readdir',
+        //'rename','unlink','rmdir',
+        'close','open','fsync',
+        'ftruncate','truncate',
+        'write','read','readFile','writeFile', 'appendFile',
+        
+        'chown','lchown','fchown',
+        'chmod','lchmod', 'fchmod',
+        'utimes','futimes',
+        'stat','lstat','fstat','exists',
+        'link','symlink','readlink','realpath',
+        
+        //'watchFile','unwatchFile','watch'
+    ].forEach(function (method) { assert(method in fs, "fs."+method+" has implementation."); });
+    
+    var BASE_DIR = "/fat_test-"+Math.random().toFixed(20).slice(2),
+        FILENAME = "Simple File.txt",
+        TEXTDATA = "Hello world!";
+    
+    var isReady = false;
+    fs.on('ready', function () {
+        assert(isReady = true, "Driver is ready.");
+    }).on('error', function (e) {
+        assert(e, "If fs driver fires 'error' event, it should include error object.");
+        assert(false, "…but driver should not error when initializing in our case.");
+    });
+    setTimeout(function () {
+        assert(isReady, "Driver fired ready event in timely fashion.");
+    }, waitTime);
+    
+    fs.readdir("/", function (e,files) {
+        assert(isReady, "Method completed after 'ready' event.");
+        assert(!e, "No error reading root directory.");
+        assert(Array.isArray(files), "Got a list of files: "+files);
+    });
+    
+    fs.mkdir(BASE_DIR, function (e) {
+        assert(!e, "No error from fs.mkdir");
+        fs.readdir(BASE_DIR, function (e,arr) {
+            assert(!e, "No error from fs.readdir");
+            assert(arr.length === 0 , "No files in BASE_DIR yet.");
+        });
+        var file = [BASE_DIR,FILENAME].join('/');
+        fs.writeFile(file, TEXTDATA, function (e) {
+            assert(!e, "No error from fs.writeFile");
+            startStreamTests();
+            fs.realpath(file, function (e,path) {
+                assert(!e, "No error from basic fs.realpath call.");
+                assert(path === file, "We already had the real path.");
+            });
+            fs.realpath([BASE_DIR,".","garbage",".","..",FILENAME].join('/'), function (e,path) {
+                assert(!e, "No error from fluffy fs.realpath call.");
+                assert(path === file, "Fixed fluffy path matches normal one.");
+            });
+            fs.realpath([BASE_DIR,"non","existent","path"].join('/'), function (e) {
+                assert(e, "Expected error calling fs.realpath on nonexistent file.");
+            });
+            
+            fs.readdir(BASE_DIR, function (e, arr) {
+                assert(!e, "Still no error from fs.readdir");
+                assert(arr.length === 2, "Test directory contains two files.");     // (ours + startStreamTests's)
+                assert(arr[0] === FILENAME, "Filename is correct.");
+                
+                fs.stat(file, function (e,d) {
+                    assert(!e, "No error from fs.stat");
+                    assert(d.isFile() === true, "Result is a file…");
+                    assert(d.isDirectory() === false, "…and not a directory.");
+                    assert(d.size === Buffer.byteLength(TEXTDATA), "Size matches length of content written.");
+                });
+                fs.exists(file, function (bool) {
+                  assert(bool === true, "File exists.");
+                });
+                fs.readFile(file, {encoding:'utf8'}, function (e, d) {
+                    assert(!e, "No error from fs.readFile");
+                    assert(d === TEXTDATA, "Data matches what was written.");
+                });
+                // now, overwrite the same file and make sure that goes well too
+                fs.writeFile(file, _.bufferFrom([0x42]), function (e) {
+                    assert(!e, "Still no error from fs.writeFile");
+                    fs.readdir(BASE_DIR, function (e, arr) {
+                        assert(!e, "No error from fs.readdir");
+                        assert(arr.length === 2, "Test directory still contains two files.");
+                        assert(arr[0] === FILENAME, "Filename still correct.");
+                        fs.stat(file, function (e,d) {
+                            assert(!e, "Still no error from fs.stat");
+                            assert(d.isFile() === true, "Result is still a file…");
+                            assert(d.isDirectory() === false, "…and not a directory.");
+                            assert(d.size === 1, "Size matches length of now-truncated content.");
+                        });
+                        fs.readFile(file, function (e, d) {
+                            assert(!e, "Still no error from fs.readFile");
+                            assert(Buffer.isBuffer(d), "Result without encoding is a buffer.");
+                            assert(d.length === 1, "Buffer is correct size.");
+                            assert(d[0] === 0x42, "Buffer content is correct.");
+                            
+                            fs.truncate(file, 1025, function (e) {
+                                assert(!e, "No error from fs.truncate (extending)");
+                                fs.readFile(file, function (e, d) {
+                                    assert(!e, "Still no error from fs.readFile after extension");
+                                    assert(d.length === 1025, "Read after extension is correct size.");
+                                    assert(d[0] === 0x42, "First byte is still correct.");
+                                    var allZeroes = true;
+                                    for (var i = 1, len = d.length; i < len; ++i) if (d[i] !== 0) allZeroes = false;
+                                    assert(allZeroes, "Extended portion of file is zero-filled.");
+                                    
+                                    fs.truncate(file, 3, function (e) {
+                                        assert(!e, "No error from fs.truncate (shortening)");
+                                        fs.readFile(file, function (e, d) {
+                                            assert(!e, "Still no error from fs.readFile after shortening.");
+                                            assert(d.length === 3, "Read after shortening is correct size.");
+                                            assert(d[0] === 0x42, "First byte is still correct.");
+                                            assert(d[1] === 0x00, "Second byte is still correct.");
+                                            assert(d[2] === 0x00, "Third byte is still correct.");
+                                            proceedWithMoreTests();
+                                        });
+                                    }); 
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        });
+        
+        
+        function proceedWithMoreTests() {
+            var fd;
+            fs.open(file, 'r', function (e, _fd) {
+                assert(!e, "No error from fs.open.");
+                fd = _fd;
+            });
+            var was = "\u0042\u0000\u0000",
+                str = "abc";
+            fs.appendFile(file, str, function (e) {
+                assert(!e, "No error from fs.appendFile.");
+                assert(fd, "File descriptor opened before appendFile called.");
+                var buf = _.allocBuffer(str.length);
+                fs.read(fd, buf, 0, buf.length, was.length, function (e,n,d) {
+                    assert(!e, "No error from fs.read after append.");
+                    assert(n === str.length, "All appended data was readable.");
+                    assert(d === buf, "Buffer returned from fs.read matched what was passed in.");
+                    assert(d.toString() === str, "Correct data found where expected.");
+                });
+            });
+            fs.readFile(file, {encoding:'ascii'}, function (e,d) {
+                assert(!e, "No error from fs.appendFile.");
+                assert(d.length === 6, "Read is correct size after append.");
+                assert(d === was+str, "Read string matches what was written and then appended.");
+                
+                fs.open(file, 'a', function (e, fd2) {
+                    assert(!e, "No error from second open of file.");
+                    var str2 = "zyx",
+                        buf2 = _.allocBuffer(str2.length+2);
+                    buf2.write(str2, 1);
+                    fs.write(fd2, buf2, 1, buf2.length-2, was.length, function (e,n,d) {
+                        assert(!e, "No error from appending fs.write.");
+                        assert(n === buf2.length-2, "Wrote proper amount from buffer.");
+                        assert(d === buf2, "Returned original buffer.");
+                        
+                        buf2.fill(0);
+                        buf2[0] = 0xFF;
+                        fs.read(fd, buf2, 1, buf2.length-1, was.length, function (e,n,d) {
+                            assert(!e, "No error from twice-appended read.");
+                            assert(n === buf2.length-1, "Read proper amount into buffer.");
+                            assert(buf2[0] === 0xFF, "Read left first byte in buffer properly alone.");
+                            assert(d.slice(1).toString() === (str+str2).slice(0, buf2.length-1), "Data was appended, not written at position.");
+                        });
+                    });
+                });
+            });
+            
+            var F = [BASE_DIR,"Manually inspect from time to time, please!.txt"].join('/'),
+                S = 512,
+                N = 16,
+                b = _.allocBuffer(S*N);
+            for (var i = 0; i < N; ++i) b.slice(S*i, S*i+S).fill(i.toString(16).charCodeAt(0));
+            fs.writeFile(F, b, function (e) {
+                assert(!e, "No error from fs.writeFile with counting blocks.");
+                fs.readFile(F, function (e,d) {
+                    assert(!e, "No error from fs.readFile with counting blocks.");
+                    assert(d.length === b.length, "Readback is correct size");
+                    var matched = true;
+                    for (var i = 0; i < S*N; ++i) if (b[i] !== d[i]) matched = false;
+                    assert(matched, "Readback matches write byte-for-byte");
+                });
+            });
+            
+            fs.stat(F, function (e,d) {
+                assert(!e, "No error from fs.stat on counting blocks file.");
+                assert(d.atime instanceof Date && !isNaN(d.atime.getTime()), "Access time is a valid date.");
+                assert(d.mtime instanceof Date && !isNaN(d.mtime.getTime()), "Modify time is a valid date.");
+                assert(d.ctime instanceof Date && !isNaN(d.ctime.getTime()), "Change^WCreate time is a valid date.");
+                var tf = d.ctime.getTime(),
+                    ct = Date.now();
+                assert(tf - 2*waitTime < ct && ct < tf + 2*waitTime, "Create time is within ± two `waitTime`s of now.");
+                fs.utimes(F, new Date(2009, 7-1, 2), null, function (e) {
+                    assert(!e, "No error from fs.utimes.");
+                    fs.stat(F, function (e,d2) {
+                        assert(!e, "No fs.stat error after touching timestamps.");
+                        assert(+d.ctime === +d2.ctime, "Create time not changed by fs.utimes");
+                        assert(d2.atime.toString().indexOf("Jul 02 2009 00:00:00") === 4, "Access time set correctly");
+                        var tf = d2.mtime.getTime(),
+                            ct = Date.now();
+                        assert(tf-2e3 < ct && ct < tf+2e3+waitTime, "Modify time is within ± a few seconds of now.");
+                        
+                        // NOTE: due to serialization, this can check results of the `fs.chmod` below, too!
+                        assert(!(d2.mode & 0100), "Archive bit is now unset.");
+                        assert(!(d2.mode & 0222), "Writable perms are unset.");
+                        assert(d2.mode & 0100000, "Regular file bit is set.");
+                    });
+                });
+                
+                assert(d.uid === 99, "Desired UID applied.");
+                assert(d.gid === 42, "Desired GID applied.");
+                assert(d.mode & 0100, "Archive bit is set.");
+                assert(d.mode & 0200, "Writable perm is set for user.");
+                assert((d.mode & 0022) === 0002, "Writable perm is only masked out for group.");
+                fs.chmod(F, 0422, function (e) {
+                    assert(!e, "No error from fs.chmod.");
+                });
+            });
+            
+            fs.chown(F, 99, 256, function (e) {
+                assert(e && e.code === 'NOSYS', "Expected error from fs.fchown.");
+            });
+        }
+        
+        function startStreamTests() {
+            var file2 = [BASE_DIR,FILENAME+"2"].join('/'),
+                outStream = fs.createWriteStream(file2);
+            var outStreamOpened = false;
+            outStream.on('open', function (fd) {
+                outStreamOpened = true;
+                assert(typeof fd === 'number', "Got file descriptor on fs.createWriteStream open.");
+            }).on('error', function (e) {
+                assert(e, "If fs.createWriteStream fires 'error' event, it should include error object.");
+                assert(false, "But, fs.createWriteStream should not error during these tests.");
+            });
+            setTimeout(function () {
+                assert(outStreamOpened, "outStream fired 'open' event in a timely fashion.");
+            }, waitTime);
+            var TEXT_MOD = TEXTDATA.toLowerCase()+"\n",
+                NUM_REPS = (waitTime <= 1e3) ? 1024 : 16;
+            outStream.write(TEXT_MOD, 'utf16le');
+            outStream.write("Ο καλύτερος χρόνος να φυτευτεί ένα \ud83c\udf31 είναι δέκα έτη πριν.", 'utf16le');
+            outStream.write("La vez del segundo mejor ahora está.\n", 'utf16le');
+            for (var i = 0; i < NUM_REPS; ++i) outStream.write("123456789\n", 'ascii');
+            outStream.write("JavaScript how do they work\n", 'utf16le');
+            outStream.write("The end, almost.\n", 'utf16le');
+            outStream.end(TEXTDATA, 'utf16le');
+            var outStreamFinished = false;
+            outStream.on('finish', function () {
+                outStreamFinished = true;
+                
+                var inStream = fs.createReadStream(file2, {start:NUM_REPS*10, encoding:'utf16le', autoClose:false}),
+                    gotData = false, gotEOF = false, inStreamFD = null;
+                inStream.on('open', function (fd) {
+                    assert(typeof fd === 'number', "Got file descriptor on fs.createReadStream open.");
+                    inStreamFD = fd;
+                });
+                inStream.on('data', function (d) {
+                    gotData = true;
+                    assert(typeof d === 'string', "Data returned as string.");
+                    assert(d.slice(d.length-TEXTDATA.length) === TEXTDATA, "End of file matches what was written.");
+                });
+                inStream.on('end', function () {
+                    gotEOF = true;
+                    
+                    var len = Buffer.byteLength(TEXT_MOD, 'utf16le'),
+                        buf = _.allocBuffer(len);
+                    fs.fsync(inStreamFD, function (e) {
+                        assert(!e, "No error from proper fsync.");
+                    });
+                    fs.fsync('garbage', function (e) {
+                        assert(e, "Expected error from garbage fsync.");
+                    });
+                    fs.read(inStreamFD, buf, 0, len, 0, function (e,n,d) {
+                        assert(!e, "No error reading from beginning of inStream's file descriptor.");
+                        assert(n === len, "Read complete buffer at beginning of inStream's fd.");
+                        assert(d.toString('utf16le') === TEXT_MOD, "Data matches at beginning of inStream's fd.");
+                        fs.close(inStreamFD, function (e) {
+                            assert(!e, "No error closing inStream's fd.");
+                        });
+                    });
+                });
+                setTimeout(function () {
+                    assert(gotData, "inStream fired 'data' event in a timely fashion.");
+                    setTimeout(function () {
+                        assert(gotEOF, "inStream fired 'eof' event in a timely fashion.");
+                    }, waitTime);
+                }, waitTime);
+            });
+            setTimeout(function () {
+                assert(outStreamFinished, "outStream fired 'finish' event in a timely fashion.");
+            }, 2*waitTime);
+        }
+    });
+}
+
+
+function assert(b,msg) { if (!msg) console.warn("no msg", Error().stack); if (!b) throw Error("Assertion failure. "+msg); else console.log(msg); }

@vates/fatfs/vol.js

@@ -0,0 +1,165 @@
+var S = require("./structs.js"),
+    c = require("./chains.js"),
+    $ = require("./cache.js"),
+    _ = require("./helpers.js");
+
+exports.init = function (volume, opts, bootSector) {
+    if (bootSector[510] !== 0x55 || bootSector[511] !== 0xAA) throw Error("Invalid volume signature!");
+    
+    var isFAT16 = bootSector.readUInt16LE(S.boot16.fields['FATSz16'].offset),
+        bootStruct = (isFAT16) ? S.boot16 : S.boot32,
+        BS = bootStruct.valueFromBytes(bootSector);
+    _.log(_.log.DBG, "Boot sector info:", BS);
+    bootSector = null;      // allow GC
+    if (!BS.BytsPerSec) throw Error("This looks like an ExFAT volume! (unsupported)");
+    else if (BS.BytsPerSec !== volume.sectorSize) throw Error("Sector size mismatch with FAT table.");
+    
+    var FATSz = (isFAT16) ? BS.FATSz16 : BS.FATSz32,
+        rootDirSectors = Math.ceil((BS.RootEntCnt * 32) / BS.BytsPerSec),
+        firstDataSector = BS.ResvdSecCnt + (BS.NumFATs * FATSz) + rootDirSectors,
+        totSec = (BS.TotSec16) ? BS.TotSec16 : BS.TotSec32,
+        dataSec = totSec - firstDataSector,
+        countofClusters = Math.floor(dataSec / BS.SecPerClus);
+    // avoid corrupting sectors from other partitions or whatnot
+    if (totSec > volume.numSectors) throw Error("Volume size mismatch!");
+    
+    var fatType;
+    if (countofClusters < 4085) {
+        fatType = 'fat12';
+    } else if (countofClusters < 65525) {
+        fatType = 'fat16';
+    } else {
+        fatType = 'fat32';
+    }
+    
+    _.log(_.log.DBG, "rootDirSectors", rootDirSectors, "firstDataSector", firstDataSector, "countofClusters", countofClusters, "=>", fatType);
+    
+    var vol = {};
+    
+    vol.opts = opts;
+    
+    vol._sectorSize = BS.BytsPerSec;
+    vol._sectorsPerCluster = BS.SecPerClus;
+    vol._firstSectorOfCluster = function (n) {
+        return firstDataSector + (n-2)*vol._sectorsPerCluster;
+    };
+    
+    vol._makeCache = function () {
+        return $.wrapDriver(volume);
+    };
+    
+    vol._readSectors = function (cache, secNum, dest, cb) {
+        if (typeof dest === 'function') {
+            cb = dest;
+            dest = _.allocBuffer(vol._sectorSize);
+        }
+        _.log(_.log.DBG, "vol._readSectors", secNum, dest.length);
+        if (secNum < volume.numSectors) cache.readSectors(secNum, dest, function (e) { cb(e, dest); });
+        else throw Error("Invalid sector number!");
+    };
+    
+    vol._writeSectors = function (cache, secNum, data, cb) {
+        _.log(_.log.DBG, "vol._writeSectors", secNum, data.length);
+        // NOTE: these are internal assertions, public API will get proper `S.err`s
+        if (data.length % volume.sectorSize) throw Error("Buffer length not a multiple of sector size");
+        else if (opts.ro) throw Error("Read-only filesystem");
+        else if (secNum < volume.numSectors) cache.writeSectors(secNum, data, cb);
+        else throw Error("Invalid sector number!");
+    };
+    
+    function fatInfoForCluster(n) {
+        var entryStruct = S.fatField[fatType],
+            FATOffset = (fatType === 'fat12') ? Math.floor(n/2) * entryStruct.size : n * entryStruct.size,
+            SecNum = BS.ResvdSecCnt + Math.floor(FATOffset / BS.BytsPerSec);
+            EntOffset = FATOffset % BS.BytsPerSec;
+        return {sector:SecNum-BS.ResvdSecCnt, offset:EntOffset, struct:entryStruct};
+    }
+    
+    // TODO: all this FAT manipulation is crazy inefficient! needs read caching *and* write caching
+    //        …the best place for cache might be in `volume` handler, though. add a `flush` method to that spec?
+    // TODO: how should we handle redundant FATs? mirror every write? just ignore completely? copy-on-eject?
+    
+    var fatChain = c.sectorChain(vol, BS.ResvdSecCnt, FATSz);
+    fatChain.cacheAdvice = 'RANDOM';
+    
+    vol.fetchFromFAT = function (clusterNum, cb) {
+        var info = fatInfoForCluster(clusterNum);
+        fatChain.readFromPosition(info, info.struct.size, function (e,n,d) {
+            if (e) return cb(e);
+            var status = info.struct.valueFromBytes(d), prefix;
+            if (fatType === 'fat12') {
+                if (clusterNum % 2) {
+                    status = (status.field1ab << 4) + status.field1c;
+                } else {
+                    status = (status.field0a << 8) + status.field0bc;
+                }
+            }
+            else if (fatType === 'fat32') {
+                status &= 0x0FFFFFFF;
+            }
+            
+            var prefix = S.fatPrefix[fatType];
+            if (status === S.fatStat.free) cb(null, 'free');
+            else if (status === S.fatStat._undef) cb(null, '-invalid-');
+            else if (status > prefix+S.fatStat.eofMin) cb(null, 'eof');
+            else if (status === prefix+S.fatStat.bad) cb(null, 'bad');
+            else if (status > prefix+S.fatStat.rsvMin) cb(null, 'reserved');
+            else cb(null, status);
+        });
+    };
+    
+    vol.storeToFAT = function (clusterNum, status, cb) {
+        if (typeof status === 'string') {
+            status = S.fatStat[status];
+            status += S.fatPrefix[fatType];
+        }
+        var info = fatInfoForCluster(clusterNum);
+        // TODO: technically fat32 needs to *preserve* the high 4 bits
+        if (fatType === 'fat12') fatChain.readFromPosition(info, info.struct.size, function (e,n,d) {
+            var value = info.struct.valueFromBytes(d);
+            if (clusterNum % 2) {
+                value.field1ab = status >>> 4;
+                value.field1c = status & 0x0F;
+            } else {
+                value.field0a = status >>> 8;
+                value.field0bc = status & 0xFF;
+            }
+            var entry = info.struct.bytesFromValue(value);
+            fatChain.writeToPosition(info, entry, cb);
+        }); else {
+            var entry = info.struct.bytesFromValue(status);
+            fatChain.writeToPosition(info, entry, cb);
+        }
+    };
+    
+    vol.allocateInFAT = function (hint, cb) {
+        if (typeof hint === 'function') {
+            cb = hint;
+            hint = 2;   // TODO: cache a better starting point?
+        }
+        function searchForFreeCluster(num, cb) {
+            if (num < countofClusters) vol.fetchFromFAT(num, function (e, status) {
+                if (e) cb(e);
+                else if (status === 'free') cb(null, num);
+                else searchForFreeCluster(num+1, cb);
+            }); else cb(S.err.NOSPC());     // TODO: try searching backwards from hint…
+        }
+        searchForFreeCluster(hint, function (e, clusterNum) {
+            if (e) cb(e);
+            else vol.storeToFAT(clusterNum, 'eof', cb.bind(null,null,clusterNum));
+        });
+    };
+    
+    vol.rootDirectoryChain = (isFAT16) ?
+        c.sectorChain(vol, firstDataSector - rootDirSectors, rootDirSectors) :
+        c.clusterChain(vol, BS.RootClus);
+    vol.rootDirectoryChain.cacheAdvice = 'WILLNEED';
+    vol.chainForCluster = c.clusterChain.bind(c, vol);
+    vol.chainFromJSON = function (d) {
+        return ('numSectors' in d) ?
+            c.sectorChain(vol, d.firstSector, d.numSectors) :
+            c.clusterChain(vol, d.firstCluster);
+    };
+    
+    return vol;
+}

@vates/fuse-vhd/.USAGE.md

@@ -13,7 +13,7 @@ await mount(handler, diskId, mountPoint)
 
 ### cli
 
-From the install folder :
+From the install folder:
 
 ```
 cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
@@ -25,4 +25,25 @@ After installing the package
 xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
 ```
 
-remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
+The `remoteUrl` can be found using the following command from the XOA CLI, and then picking the right `url` property:
+
+`xo-server-db ls remote`
+
+The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
+
+## Restore a file from a VHD using `fuse-vhd` CLI
+
+1. Mount a VHD to the Filesystem (see **Usage**).
+2. Verify the VHD is Correctly Mounted:
+
+`mount | grep fuse`
+
+3. Get the `START` and `SIZE` of the disk whose file you want to restore. Multiply `START` by 512 (block size) to get the offset value. (`mountedVhdPath` is the `mountPoint` value from earlier, followed by `/vhd0`, for example). Depending on the partition type, you may need to do some additional setup to discover the partitions.
+
+`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
+
+4.  Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
+
+`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512>  --source=<mountedVhdPath> --target=<diskMountPoint>`
+
+5. Copy your files from the mounted partition, then unmount the partition.

@vates/fuse-vhd/README.md

@@ -29,7 +29,7 @@ await mount(handler, diskId, mountPoint)
 
 ### cli
 
-From the install folder :
+From the install folder:
 
 ```
 cli.mjs <remoteUrl> <vhdPathInRemote> <mountPoint>
@@ -41,7 +41,28 @@ After installing the package
 xo-fuse-vhd <remoteUrl> <vhdPathInRemote> <mountPoint>
 ```
 
-remoteUrl can be found by using cli in `@xen-orchestra/fs` , for example a local remote will have a url like `file:///path/to/remote/root`
+The `remoteUrl` can be found using the following command from the XOA CLI, and then picking the right `url` property:
+
+`xo-server-db ls remote`
+
+The `vhdPathInRemote` follows a file structure described in [VM backups](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/docs/VM%20backups/README.md).
+
+## Restore a file from a VHD using `fuse-vhd` CLI
+
+1. Mount a VHD to the Filesystem (see **Usage**).
+2. Verify the VHD is Correctly Mounted:
+
+`mount | grep fuse`
+
+3. Get the `START` and `SIZE` of the disk whose file you want to restore. Multiply `START` by 512 (block size) to get the offset value. (`mountedVhdPath` is the `mountPoint` value from earlier, followed by `/vhd0`, for example). Depending on the partition type, you may need to do some additional setup to discover the partitions.
+
+`partx --bytes --output=NR,START,SIZE,NAME,UUID,TYPE --pairs <mountedVhdPath>`
+
+4.  Mount the disk. Depending on the partition type, you may need to add some additional options. The `norecovery` option is used for ext3/ext4/xfs file systems; otherwise, remove this option.
+
+`mount --options=loop,ro,norecovery,sizelimit=<SIZE>,offset=<START*512>  --source=<mountedVhdPath> --target=<diskMountPoint>`
+
+5. Copy your files from the mounted partition, then unmount the partition.
 
 ## Contributions
 

@vates/fuse-vhd/package.json

@@ -19,12 +19,12 @@
   },
   "main": "./index.mjs",
   "dependencies": {
-    "@xen-orchestra/fs": "^4.2.1",
+    "@xen-orchestra/fs": "^4.6.6",
     "@xen-orchestra/log": "^0.7.1",
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.11.1"
+    "vhd-lib": "^4.14.7"
   },
   "bin": {
     "xo-fuse-vhd": "./cli.mjs"

@vates/generator-toolbox/.USAGE.md

@@ -0,0 +1,58 @@
+A toolbox to ease the use of generator
+
+### Timeout
+
+wrap a source async generator to have it throw an error when timeout is reached
+timeout is a positive number in milliseconds
+
+```js
+import { Timeout } from '@vates/generator-toolbox'
+
+const wrappedGenerator = new Timeout(sourceGenerator, timeout)
+```
+
+### Throttle
+
+wrap a source async generator to have it respect a max speed ( in bytes per seconds).
+speed is either a strictly positive number or a function returning a strictly positive number. A speed change will be used for the next emitted packet.
+
+The source generator must yield object with a length property.
+
+Optimized for small yields regarding to the speed, since it won't split incoming packet.
+
+If the generator reached the max speed it will be paused, limiting memory consumption.
+
+```js
+import { Throttle } from '@vates/generator-toolbox'
+
+const wrappedGenerator = new Throttle(sourceGenerator, speed)
+```
+
+### Synchronized
+
+Fork a generator. The rules are:
+
+- if the source returns, all the forks return
+- if the forks error, all the forks error with the same error
+- if a fork returns, it is stopped, but the generator continue with the other
+- if a fork errors, it is stopped, but the generator continue with the other
+- if all the forks return, the source is stopped
+- if all the forks error, the source is errored with the last error
+- the source start producing a packet when the fastest forked ask for it
+- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
+
+```ts
+import { Synchronized } from '@vates/generator-toolbox'
+
+async function consume(generator: AsyncGenerator) {
+  for await (const val of generator) {
+    console.log({ val })
+  }
+}
+const forker = new Synchronized(generator)
+const first = forker.fork('first')
+const second = forker.fork('second')
+await Promise.all([consume(first), consume(second)])
+```
+
+Note: you can stop early a generator by calling `generator.return()`, and you can stop in in error by calling `generator.throw(error)`

@vates/generator-toolbox/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/generator-toolbox/README.md

@@ -0,0 +1,89 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/generator-toolbox
+
+[![Package Version](https://badgen.net/npm/v/@vates/generator-toolbox)](https://npmjs.org/package/@vates/generator-toolbox) ![License](https://badgen.net/npm/license/@vates/generator-toolbox) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/generator-toolbox)](https://bundlephobia.com/result?p=@vates/generator-toolbox) [![Node compatibility](https://badgen.net/npm/node/@vates/generator-toolbox)](https://npmjs.org/package/@vates/generator-toolbox)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/generator-toolbox):
+
+```sh
+npm install --save @vates/generator-toolbox
+```
+
+## Usage
+
+A toolbox to ease the use of generator
+
+### Timeout
+
+wrap a source async generator to have it throw an error when timeout is reached
+timeout is a positive number in milliseconds
+
+```js
+import { Timeout } from '@vates/generator-toolbox'
+
+const wrappedGenerator = new Timeout(sourceGenerator, timeout)
+```
+
+### Throttle
+
+wrap a source async generator to have it respect a max speed ( in bytes per seconds).
+speed is either a strictly positive number or a function returning a strictly positive number. A speed change will be used for the next emitted packet.
+
+The source generator must yield object with a length property.
+
+Optimized for small yields regarding to the speed, since it won't split incoming packet.
+
+If the generator reached the max speed it will be paused, limiting memory consumption.
+
+```js
+import { Throttle } from '@vates/generator-toolbox'
+
+const wrappedGenerator = new Throttle(sourceGenerator, speed)
+```
+
+### Synchronized
+
+Fork a generator. The rules are:
+
+- if the source returns, all the forks return
+- if the forks error, all the forks error with the same error
+- if a fork returns, it is stopped, but the generator continue with the other
+- if a fork errors, it is stopped, but the generator continue with the other
+- if all the forks return, the source is stopped
+- if all the forks error, the source is errored with the last error
+- the source start producing a packet when the fastest forked ask for it
+- the source forks get the packet only when all the forks asked for it, no buffer stores in memory
+
+```ts
+import { Synchronized } from '@vates/generator-toolbox'
+
+async function consume(generator: AsyncGenerator) {
+  for await (const val of generator) {
+    console.log({ val })
+  }
+}
+const forker = new Synchronized(generator)
+const first = forker.fork('first')
+const second = forker.fork('second')
+await Promise.all([consume(first), consume(second)])
+```
+
+Note: you can stop early a generator by calling `generator.return()`, and you can stop in in error by calling `generator.throw(error)`
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[MIT](https://spdx.org/licenses/MIT) © [Vates SAS](https://vates.fr)

@vates/generator-toolbox/eslint.config.mjs

@@ -0,0 +1,6 @@
+// @ts-check
+
+import eslint from '@eslint/js'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended)

@vates/generator-toolbox/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@vates/generator-toolbox",
+  "version": "1.1.0",
+  "main": "dist/index.mjs",
+  "license": "MIT",
+  "private": false,
+  "type": "module",
+  "devDependencies": {
+    "@eslint/js": "^9.19.0",
+    "rimraf": "^6.0.1",
+    "@types/node": "^20.6",
+    "typescript": "~5.6",
+    "typescript-eslint": "^8.23.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "postversion": "npm publish --access public",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "test": "tsc && node --test **/*.test.mjs",
+    "clean": "rimraf dist/"
+  },
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/generator-toolbox",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/generator-toolbox",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "engines": {
+    "node": ">=20.18"
+  }
+}

@vates/generator-toolbox/src/index.mts

@@ -0,0 +1,3 @@
+export { Synchronized } from './synchronized.mjs'
+export { Throttle } from './throttle.mjs'
+export { Timeout } from './timeout.mjs'

@vates/generator-toolbox/src/synchronized.mts

@@ -0,0 +1,157 @@
+import assert from 'node:assert'
+
+export class Synchronized<T, TReturn, TNext> {
+  #source: AsyncGenerator<T, TReturn, TNext>
+  #forks = new Map<string, Forked<T, TReturn, TNext>>()
+  #removedForks = new Set<string>()
+  #waitingForks = new Set<string>()
+  #started = false
+
+  #nextValueForksReady?: {
+    promise: Promise<IteratorResult<T>>
+    forksWaitingReject: (error: Error) => void
+    forksWaitingResolve: () => void
+  }
+
+  constructor(source: AsyncGenerator<T, TReturn, TNext>) {
+    this.#source = source
+  }
+
+  fork(uid: string): AsyncGenerator<T, TReturn, TNext> {
+    assert.strictEqual(this.#started, false, `can't create a fork after consuming the data`)
+    assert.strictEqual(this.#forks.has(uid), false, `a fork with id ${uid} already exists`)
+
+    const fork = new Forked<T, TReturn, TNext>(this, uid)
+    this.#forks.set(uid, fork)
+    return fork
+  }
+
+  async #resolveWhenAllForksReady(): Promise<IteratorResult<T>> {
+    if (!this.#nextValueForksReady) {
+      throw new Error('Can t wait forks if there are noone waiting')
+    }
+    const { promise, forksWaitingResolve } = this.#nextValueForksReady
+    if (this.#waitingForks.size === this.#forks.size) {
+      // reset value
+      this.#waitingForks.clear()
+      this.#nextValueForksReady = undefined
+      forksWaitingResolve() // for the other forks waiting
+    }
+    return promise
+  }
+
+  async next(uid: string): Promise<IteratorResult<T>> {
+    if (this.#removedForks.has(uid)) {
+      return { done: true, value: undefined }
+    }
+    if (!this.#forks.has(uid)) {
+      throw new Error(`trying to advance fork ${uid} that is not a fork of this one`)
+    }
+
+    if (this.#waitingForks.has(uid)) {
+      throw new Error(`Fork ${uid} is already waiting`)
+    }
+
+    this.#started = true
+    if (this.#nextValueForksReady === undefined) {
+      let forksWaitingResolve = () => {}
+      let forksWaitingReject: (reason?: Error) => void = () => {}
+      const next = this.#source.next().catch(async error => {
+        const e = new Error(`Error in the source generator ${error.message}`, { cause: error })
+        forksWaitingReject(e)
+        // source has failed, kill everything, and stop the forks
+        for (const uid of [...this.#forks.keys()]) {
+          await this.remove(uid, error)
+        }
+      })
+      const promise = Promise.all([
+        next,
+        new Promise((_resolve, _reject) => {
+          forksWaitingResolve = () => _resolve(undefined)
+          forksWaitingReject = _reject
+        }),
+      ]).then(([_]) => _ as IteratorResult<T>)
+
+      this.#nextValueForksReady = { promise, forksWaitingResolve, forksWaitingReject }
+    }
+    this.#waitingForks.add(uid)
+    return this.#resolveWhenAllForksReady()
+  }
+
+  async remove(uid: string, error?: Error): Promise<IteratorResult<T>> {
+    const fork = this.#forks.get(uid)
+    if (fork === undefined) {
+      if (this.#removedForks.has(uid)) {
+        // already removed
+        return { done: true, value: undefined }
+      }
+      throw new Error(`trying to remove fork with uid ${uid} that is not a fork of this one`)
+    }
+    this.#forks.delete(uid)
+    this.#waitingForks.delete(uid)
+    this.#removedForks.add(uid)
+    try {
+      if (error === undefined) {
+        await fork.return()
+      } else {
+        await fork.throw(error)
+      }
+    } catch (cleaningError) {
+      console.error('Error while cleaning the forked', {
+        cleaningError,
+        sourceError: error,
+      })
+    }
+
+    if (this.#forks.size === 0) {
+      if (error === undefined) {
+        await this.#source.return(undefined as TReturn)
+      } else {
+        await this.#source.throw(error)
+      }
+      // Reject any pending forks waiting for the next value
+      if (this.#nextValueForksReady) {
+        this.#nextValueForksReady.forksWaitingReject(new Error('Source generator terminated.', { cause: error }))
+        this.#nextValueForksReady = undefined
+      }
+      // clear state
+      this.#removedForks.clear()
+      this.#waitingForks.clear()
+    } else {
+      // this fork was maybe blocking the others
+      if (this.#nextValueForksReady) {
+        await this.#resolveWhenAllForksReady()
+      }
+    }
+    return { done: true, value: undefined }
+  }
+}
+
+class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
+  #parent: Synchronized<T, TReturn, TNext>
+  #uid: string
+  constructor(parent: Synchronized<T, TReturn, TNext>, uid: string) {
+    this.#parent = parent
+    this.#uid = uid
+  }
+  next(): Promise<IteratorResult<T>> {
+    return this.#parent.next(this.#uid)
+  }
+  async return(): Promise<IteratorResult<T>> {
+    return this.#parent.remove(this.#uid)
+  }
+  async throw(e: Error): Promise<IteratorResult<T>> {
+    return this.#parent.remove(this.#uid, e)
+  }
+
+  async *[Symbol.asyncIterator](): AsyncGenerator<T> {
+    while (true) {
+      const res = await this.next()
+      if (res.done) {
+        break
+      }
+
+      yield res.value
+    }
+  }
+}

@vates/generator-toolbox/src/synchronized.test.mts

@@ -0,0 +1,150 @@
+import assert from 'node:assert'
+import { suite, test } from 'node:test'
+import { Synchronized } from './synchronized.mjs'
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (val: unknown) => {}) {
+  for (let i = 0; i < end; i++) {
+    await new Promise(resolve => setTimeout(resolve, 10))
+    yield i
+    onYielded(i)
+    progress.yielded = i
+  }
+  return progress
+}
+
+async function consume(
+  iterable: AsyncGenerator,
+  delay = 2500,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  onConsumed = (val: unknown, iterable: AsyncGenerator) => Promise.resolve(false)
+) {
+  for await (const val of iterable) {
+    await new Promise(resolve => setTimeout(resolve, delay))
+    if (await onConsumed(val, iterable)) {
+      iterable.return(undefined)
+    }
+  }
+}
+
+suite('success', () => {
+  test('if works with multiple consumer', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(3, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+    await Promise.all([consume(first, 50), consume(second, 500)])
+    assert.strictEqual(progress.yielded, 2)
+  })
+
+  test('if works with one consumer returning', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(10, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+    await new Promise(resolve => setTimeout(resolve, 500))
+    await Promise.all([
+      consume(first, 1),
+      consume(second, 1, async (val: unknown) => {
+        return Promise.resolve(val === 2)
+      }),
+    ])
+    assert.strictEqual(progress.yielded, 9)
+  })
+  test('if works with all consumers returning', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(10, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+    await new Promise(resolve => setTimeout(resolve, 500))
+    await Promise.all([
+      consume(first, 1, async (val: unknown) => {
+        return Promise.resolve(val === 5)
+      }),
+      consume(second, 1, async (val: unknown) => {
+        return Promise.resolve(val === 2)
+      }),
+    ])
+    assert.strictEqual(progress.yielded, 4)
+  })
+})
+
+suite('error', () => {
+  test('it fails all if error on source', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(5, progress, (val: unknown) => {
+      if (val === 2) {
+        throw new Error('source stop')
+      }
+    })
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+    await assert.rejects(Promise.all([consume(first, 1), consume(second, 1)]))
+
+    assert.strictEqual(progress.yielded, 1)
+  })
+  test('it flows until the end if one consumer throws', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(10, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+
+    const firstConsume = consume(first, 1)
+
+    const secondConsume = assert.rejects(() =>
+      consume(second, 1, async (val: unknown, iterable) => {
+        if (val === 1) {
+          await iterable.throw(new Error('error on second'))
+        }
+        return false
+      })
+    )
+    await Promise.allSettled([firstConsume, secondConsume])
+    assert.strictEqual(progress.yielded, 9)
+  })
+  test('it throw if ALL consumer throws', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(5, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    const second = forker.fork('second')
+
+    const firstConsume = consume(first, 1, async (val: unknown, iterable) => {
+      if (val === 3) {
+        await iterable.throw(new Error('error on first'))
+      }
+      return false
+    })
+    const secondConsume = consume(second, 1, async (val: unknown, iterable) => {
+      if (val === 1) {
+        await iterable.throw(new Error('error on second'))
+      }
+      return false
+    })
+
+    await assert.rejects(Promise.all([firstConsume, secondConsume]))
+    assert.strictEqual(progress.yielded, 2)
+  })
+
+  test('It should not allow a new fork after the start', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(5, progress)
+    const forker = new Synchronized(generator)
+    const first = forker.fork('first')
+    await first.next()
+    assert.throws(() => forker.fork('second'))
+  })
+  test('It should not allow fork with the same uid', async () => {
+    const progress = { yielded: 0 }
+    const generator = makeRangeGenerator(5, progress)
+    const forker = new Synchronized(generator)
+    forker.fork('first')
+    forker.fork('second')
+    assert.throws(() => forker.fork('first'))
+  })
+})

@vates/generator-toolbox/src/throttle.mts

@@ -0,0 +1,68 @@
+import assert from 'node:assert'
+
+/**
+ *
+ * This class will throttle the production of a group of generators
+ *
+ * limits: depends on the event loop, so don't expect it to be very precise
+ * works better with more small blocks
+ * The source generator must yield object with a length property
+ * Changing the speed will only be takin into account fot the next packets asked
+ */
+export class Throttle {
+  #previousSlot = 0
+  #bytesPerSecond: number | (() => number)
+  get speed(): number {
+    let speed: number
+    if (typeof this.#bytesPerSecond === 'function') {
+      speed = this.#bytesPerSecond()
+    } else {
+      speed = this.#bytesPerSecond
+    }
+    assert.ok(speed >= 0, `speed must be positive, ${speed} computed`)
+    return speed
+  }
+  constructor(speed: number | (() => number)) {
+    this.#bytesPerSecond = speed
+  }
+
+  getNextSlot(length: number): { timeout?: ReturnType<typeof setTimeout>; promise?: Promise<unknown> } {
+    assert.notStrictEqual(length, undefined, `throttled stream need to expose a length property }`)
+    assert.ok(length > 0, `throttled stream must expose a positive length property , ${length} given }`)
+    if (this.speed === 0) {
+      return {}
+    }
+    const previous = this.#previousSlot
+    const nextSlot = Math.round(previous + (length * 1000) / this.speed)
+    if (nextSlot < Date.now()) {
+      // we're above the limit, go now
+      this.#previousSlot = Date.now()
+      return {}
+    }
+    this.#previousSlot = nextSlot
+    // wait till the next slot
+    // it won't be extremely precise since the event loop is not
+    let timeout: ReturnType<typeof setTimeout> | undefined = undefined
+    const promise = new Promise(function (resolve) {
+      timeout = setTimeout(resolve, nextSlot - Date.now())
+    })
+    return { promise, timeout }
+  }
+
+  async *createThrottledGenerator(source: AsyncGenerator<{ length: number }>): AsyncGenerator<{ length: number }> {
+    let timeout: ReturnType<typeof setTimeout> | undefined
+    try {
+      for await (const value of source) {
+        const res = this.getNextSlot(value.length)
+        timeout = res.timeout
+        // wait for the time slot before yielding the data
+        if (res.promise !== undefined) {
+          await res.promise
+        }
+        yield value
+      }
+    } finally {
+      clearTimeout(timeout)
+    }
+  }
+}

@vates/generator-toolbox/src/throttle.test.mts

@@ -0,0 +1,63 @@
+import { Throttle } from './throttle.mjs'
+import { suite, test } from 'node:test'
+import assert from 'node:assert'
+
+async function* makeGenerator(chunkSize = 1024 * 1024, nbChunks = 100) {
+  for (let i = 0; i < nbChunks; i++) {
+    yield Buffer.allocUnsafe(chunkSize)
+  }
+}
+
+async function consumes(generator: AsyncGenerator) {
+  // eslint-disable-next-line no-empty,@typescript-eslint/no-unused-vars
+  for await (const data of generator) {
+  }
+}
+
+suite('it throttle one', () => {
+  test('base test', async () => {
+    // 10MB/s
+    const throttler = new Throttle(10 * 1024 * 1024)
+    const throttled = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 20))
+
+    const start = Date.now()
+    await consumes(throttled)
+    const end = Date.now()
+    assert.strictEqual(Math.round((end - start) / 1000), 2)
+  })
+  test('variable speed test', async () => {
+    // 10MB/s
+    const throttler = new Throttle(() => 10 * 1024 * 1024)
+    const throttled = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 20))
+
+    const start = Date.now()
+    await consumes(throttled)
+    const end = Date.now()
+    assert.strictEqual(Math.round((end - start) / 1000), 2)
+  })
+  test('multiple generator test', async () => {
+    // 10MB/s
+    const throttler = new Throttle(10 * 1024 * 1024)
+    const first = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
+    const second = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
+    const start = Date.now()
+
+    await Promise.all([consumes(first), consumes(second)])
+    const end = Date.now()
+    assert.strictEqual(Math.round((end - start) / 1000), 2)
+  })
+  test(`it doesn't accept negative value`, async () => {
+    // 10MB/s
+    let throttler = new Throttle(-10 * 1024 * 1024)
+    let generator = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
+    await assert.rejects(() => consumes(generator))
+
+    let index = 5
+    throttler = new Throttle(() => {
+      index--
+      return index > 0 ? 10 * 1024 * 1024 : -10 * 1024 * 1024
+    })
+    generator = throttler.createThrottledGenerator(makeGenerator(1024 * 1024, 10))
+    await assert.rejects(() => consumes(generator))
+  })
+})

@vates/generator-toolbox/src/timeout.mts

@@ -0,0 +1,37 @@
+import assert from 'node:assert'
+export class Timeout<T> implements AsyncGenerator {
+  #source: AsyncGenerator<T>
+  #timeout: number
+  constructor(source: AsyncGenerator<T>, timeout: number) {
+    assert.ok(timeout > 0, `Timeout value must be positive, ${timeout} received`)
+    this.#source = source
+    this.#timeout = timeout
+  }
+  async next(): Promise<IteratorResult<T>> {
+    let timeout: ReturnType<typeof setTimeout>
+    const promiseTimeout = new Promise((_, reject) => {
+      timeout = setTimeout(() => {
+        reject(new Error('Timeout reached '))
+      }, this.#timeout)
+    })
+    const promiseNext = new Promise((resolve, reject) => {
+      this.#source.next().then(res => {
+        // ensure timetout won't fire later
+        clearTimeout(timeout)
+        resolve(res)
+      }, reject)
+    })
+    // promiseTimeout will never resolve
+    return Promise.race([promiseNext, promiseTimeout]) as Promise<IteratorResult<T>>
+  }
+  return(): Promise<IteratorResult<T>> {
+    return this.#source.return(undefined)
+  }
+  throw(e: Error): Promise<IteratorResult<T>> {
+    return this.#source.throw(e)
+  }
+
+  [Symbol.asyncIterator](): AsyncGenerator<T> {
+    return this
+  }
+}

@vates/generator-toolbox/src/timeout.test.mts

@@ -0,0 +1,67 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
+import { Timeout } from './timeout.mjs'
+
+describe('Timeout class', () => {
+  it('should reject a timeout with a negative or zero value ', async () => {
+    // Create a mock async generator that never resolves
+    const mockGenerator = (async function* () {
+      yield 1
+    })()
+
+    assert.throws(() => new Timeout(mockGenerator, 0))
+    assert.throws(() => new Timeout(mockGenerator, -10))
+  })
+  it('should resolve with the value from the source generator if it completes before the timeout', async () => {
+    // Create a mock async generator that resolves immediately
+    const mockGenerator = (async function* () {
+      yield 1
+    })()
+
+    const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
+
+    const result = await timeout.next()
+    assert.strictEqual(result.value, 1)
+    assert.strictEqual(result.done, false)
+  })
+
+  it('should reject with a timeout error if the source generator takes too long', async () => {
+    // Create a mock async generator that never resolves
+    // eslint-disable-next-line require-yield
+    const mockGenerator = (async function* () {
+      await new Promise(() => {}) // Never resolves
+    })()
+
+    const timeout = new Timeout(mockGenerator, 10) // 10ms timeout
+
+    await assert.rejects(timeout.next(), 'Expected timeout error was not thrown')
+  })
+  it('should allow iteration using for-await-of', async () => {
+    // Create a mock async generator that yields values
+    const mockGenerator: AsyncGenerator<number> = (async function* () {
+      yield 1
+      yield 2
+      yield 3
+    })()
+
+    const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
+
+    const results: Array<number> = []
+    for await (const value of timeout) {
+      results.push(value as number)
+    }
+
+    assert.deepStrictEqual(results, [1, 2, 3])
+  })
+
+  it('should handle the throw method correctly', async () => {
+    // Create a mock async generator with a throw method
+    const mockGenerator = (async function* () {
+      yield 1
+    })()
+
+    const timeout = new Timeout(mockGenerator, 100) // 100ms timeout
+
+    await assert.rejects(timeout.throw(new Error('Test error')), 'Expected error was not thrown')
+  })
+})

@vates/generator-toolbox/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "es2023",
+    "module": "ESNext",
+    "types": ["node"], // Only use types from Node.js and packages that are explicitly imported
+    "moduleResolution": "bundler",
+    "strict": true,
+    "declaration": true, // Generates `.d.ts` files for type safety
+    "outDir": "dist" // Outputs compiled code to the `dist` directory
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

@vates/json-hash/.USAGE.md

@@ -1,4 +1,4 @@
-The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
+The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
 
 The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
 

@vates/json-hash/README.md

@@ -16,7 +16,7 @@ npm install --save @vates/json-hash
 
 ## Usage
 
-The use for this librariy is to create a composite key from a JSON value, ignoring objects identity and properties order.
+The use for this library is to create a composite key from a JSON value, ignoring objects identity and properties order.
 
 The hash algorithm used is intended to be fast and with low collisions and is not guaranteed to be be secure.
 

@vates/nbd-client/NbdDisk.mjs

@@ -0,0 +1,132 @@
+import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
+import NbdClient from '@vates/nbd-client'
+
+/**
+ * @typedef {{ offset: number, length: number }} DataRange
+ */
+
+/**
+ * @extends {RandomAccessDisk}
+ */
+export class NbdDisk extends RandomAccessDisk {
+  #nbdInfos
+  /** @type {NbdClient|undefined} */
+  #nbdClient
+
+  /** @type {Array<DataRange> | undefined} */
+  #dataMap
+
+  /** @type {number} */
+  #blockSize
+
+  constructor(nbdInfos, blockSize, { dataMap } = {}) {
+    super()
+    this.#blockSize = blockSize
+    this.#nbdInfos = nbdInfos
+    this.#dataMap = dataMap && this.#processDatamap(dataMap)
+  }
+
+  #processDatamap(rawDataMap) {
+    return rawDataMap
+      .filter(({ type }) => type === 0)
+      .map(({ offset, length }) => ({ offset, length }))
+      .sort(({ offset: offset1 }, { offset: offset2 }) => offset1 - offset2)
+  }
+  /**
+   * @param {number} index
+   * @returns {Promise<DiskBlock>}
+   */
+  async readBlock(index) {
+    if (!this.#nbdClient) {
+      throw new Error("can't readBlock before init")
+    }
+    let data = await this.#nbdClient.readBlock(index, this.getBlockSize())
+    if (data.length !== this.getBlockSize()) {
+      // this condition only work on non aligned disks
+      // we won't accept truncated block from unaligned disks
+      if (index === Math.ceil(this.getVirtualSize() / this.getBlockSize()) - 1) {
+        // last block add zeros at the end
+        data = Buffer.concat([data], this.getBlockSize())
+      } else {
+        throw new Error(`Block ${index} is not at the right size, expecting ${this.getBlockSize()}, got ${data.length}`)
+      }
+    }
+    return {
+      index,
+      data,
+    }
+  }
+
+  /**
+   * @returns {number}
+   */
+  getVirtualSize() {
+    if (!this.#nbdClient) {
+      throw new Error("can't get size before init")
+    }
+    return Number(this.#nbdClient.exportSize)
+  }
+
+  /**
+   * @returns {number}
+   */
+  getBlockSize() {
+    return this.#blockSize
+  }
+
+  /**
+   * @returns {Promise<void>}
+   */
+  async init() {
+    this.#nbdClient = new NbdClient(this.#nbdInfos)
+    await this.#nbdClient.connect()
+    if (this.#dataMap === undefined) {
+      this.#dataMap = this.#processDatamap(await this.#nbdClient.getMap())
+    }
+  }
+
+  /**
+   * @returns {Promise<void>}
+   */
+  async close() {
+    await this.#nbdClient?.disconnect()
+  }
+
+  /**
+   * @returns {boolean}
+   */
+  isDifferencing() {
+    return false
+  }
+
+  /**
+   * @returns {Array<number>}
+   */
+  getBlockIndexes() {
+    const indexes = new Set()
+    this.#dataMap?.forEach(({ offset, length }) => {
+      if (length === 0) return
+
+      const firstBlockIndex = Math.floor(offset / this.getBlockSize())
+      const lastBlockIndex = Math.floor((offset + length - 1) / this.getBlockSize())
+
+      for (let blockIndex = firstBlockIndex; blockIndex <= lastBlockIndex; blockIndex++) {
+        indexes.add(blockIndex)
+      }
+    })
+    return [...indexes].sort((a, b) => a - b)
+  }
+
+  /**
+   * @param {number} index
+   * @returns {boolean}
+   */
+  hasBlock(index) {
+    if (!this.#dataMap) {
+      throw new Error("can't hasBlock before init")
+    }
+    const blockStart = index * this.getBlockSize()
+    const blockEnd = (index + 1) * this.getBlockSize()
+    return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && offset < blockEnd)
+  }
+}

@vates/nbd-client/constants.mjs

@@ -1,6 +1,6 @@
 export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
 export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
-export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
+export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negotiation
 export const NBD_OPT_EXPORT_NAME = 1
 export const NBD_OPT_ABORT = 2
 export const NBD_OPT_LIST = 3

@vates/nbd-client/index.mjs

@@ -1,7 +1,7 @@
 import assert from 'node:assert'
 import { Socket } from 'node:net'
 import { connect } from 'node:tls'
-import { fromCallback, pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
+import { pRetry, pDelay, pTimeout, pFromCallback } from 'promise-toolbox'
 import { readChunkStrict } from '@vates/read-chunk'
 import { createLogger } from '@xen-orchestra/log'
 import {
@@ -20,6 +20,8 @@ import {
   OPTS_MAGIC,
   NBD_CMD_DISC,
 } from './constants.mjs'
+import { spawn } from 'node:child_process'
+
 const { warn } = createLogger('vates:nbd-client')
 
 // documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
@@ -42,9 +44,9 @@ export default class NbdClient {
   // AFAIK, there is no guaranty the server answers in the same order as the queries
   // so we handle a backlog of command waiting for response and handle concurrency manually
 
-  #waitingForResponse // there is already a listenner waiting for a response
+  #waitingForResponse // there is already a listener waiting for a response
   #nextCommandQueryId = BigInt(0)
-  #commandQueryBacklog // map of command waiting for an response queryId => { size/*in byte*/, resolve, reject}
+  #commandQueryBacklog // map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
   #connected = false
 
   #reconnectingPromise
@@ -195,7 +197,7 @@ export default class NbdClient {
     }
 
     // send export name we want to access.
-    // it's  implictly closing the negociation phase.
+    // it's implicitly closing the negotiation phase.
     await this.#write(OPTS_MAGIC)
     await this.#writeInt32(NBD_OPT_EXPORT_NAME)
     const exportNameBuffer = Buffer.from(this.#exportName)
@@ -218,8 +220,22 @@ export default class NbdClient {
   }
 
   #write(buffer) {
-    const promise = fromCallback.call(this.#serverSocket, 'write', buffer)
-    return pTimeout.call(promise, this.#messageTimeout)
+    let timeout
+    const messageTimeout = this.#messageTimeout
+    const socket = this.#serverSocket
+    return Promise.race([
+      new Promise((resolve, reject) => {
+        timeout = setTimeout(() => {
+          reject(new Error('timeout'))
+        }, messageTimeout)
+      }),
+      new Promise(resolve =>
+        socket.write(buffer, () => {
+          clearTimeout(timeout)
+          resolve()
+        })
+      ),
+    ])
   }
 
   async #readInt32() {
@@ -343,4 +359,40 @@ export default class NbdClient {
       },
     })
   }
+
+  /**
+   *  returns the map of the file with holes, zeros and data, useful to handle efficiently sparse source
+   * to implement this internally: use structure response if the server supports it, and then ask for BLOCK_STATUS   *
+   *
+   * @returns {Promise<{ offset: number, length: number, type: number }[]>}
+   * A promise that resolves to an array where each object represents a segment:
+   * - `offset` — The byte offset from the start.
+   * - `length` — The size of the segment in bytes.
+   * - `type` — A numeric code indicating the segment type ( 0 means data).
+   */
+  /* async */ getMap() {
+    return new Promise((resolve, reject) => {
+      const process = spawn('nbdinfo', [
+        '--json',
+        '--map',
+        `nbd://${this.#serverAddress}:${this.#serverPort}/${encodeURIComponent(this.#exportName)}`,
+      ])
+      let text = ''
+      let errText = ''
+      process.stdout.on('data', data => (text += data))
+      process.stderr.on('data', data => (errText += data))
+      process.on('error', reject)
+      process.on('close', code => {
+        if (code !== 0) {
+          return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
+        }
+        try {
+          const json = JSON.parse(text)
+          resolve(json)
+        } catch (error) {
+          reject(new Error(`${error} ${text}`))
+        }
+      })
+    })
+  }
 }

@vates/nbd-client/multi.mjs

@@ -30,6 +30,8 @@ export default class MultiNbdClient {
    * open nbdConcurrency connections to NBD servers
    * it must obtain at least one connection to succeed
    * it tries to spread connections on multiple host
+   *
+   * @returns {Promise<void>}
    */
   async connect() {
     const candidates = [...this.#settings]
@@ -68,7 +70,9 @@ export default class MultiNbdClient {
       await _connect()
     }
     if (this.#clients.length === 0) {
-      throw new Error(`Fail to connect to any Nbd client`)
+      const error = new Error(`Fail to connect to any Nbd client`, { nbdInfos: this.#settings })
+      error.code = 'NO_NBD_AVAILABLE'
+      throw error
     }
     if (this.#clients.length < this.#nbdConcurrency) {
       warn(
@@ -77,17 +81,30 @@ export default class MultiNbdClient {
     }
   }
 
+  /**
+   * @returns {Promise<void>}
+   */
   async disconnect() {
     await asyncEach(this.#clients, client => client.disconnect(), {
       stopOnError: false,
     })
   }
 
+  /**
+   *
+   * @param {number} index
+   * @param {number} size
+   * @returns {Promise<Buffer>}
+   */
   async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
     const clientId = index % this.#clients.length
     return this.#clients[clientId].readBlock(index, size)
   }
 
+  /**
+   *
+   * @param {AsyncGenerator<Buffer>} indexGenerator
+   */
   async *readBlocks(indexGenerator) {
     // default : read all blocks
     const readAhead = []
@@ -112,4 +129,17 @@ export default class MultiNbdClient {
       yield readAhead.shift()
     }
   }
+  /**
+   *  returns the map of the file with holes, zeros and data, useful to handle efficiently sparse source   *
+   *
+   * @returns {Promise<{ offset: number, length: number, type: number }[]>}
+   * A promise that resolves to an array where each object represents a segment:
+   * - `offset` — The byte offset from the start.
+   * - `length` — The size of the segment in bytes.
+   * - `type` — A numeric code indicating the segment type (0 means no data).
+   */
+  async getMap() {
+    // ask the map from one of the connected client
+    return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap()
+  }
 }

@vates/nbd-client/package.json

@@ -13,18 +13,19 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "3.1.2",
+  "version": "3.2.3",
   "engines": {
     "node": ">=14.0"
   },
   "main": "./index.mjs",
   "dependencies": {
-    "@vates/async-each": "^1.0.0",
-    "@vates/read-chunk": "^1.2.0",
+    "@vates/async-each": "^1.0.1",
+    "@vates/read-chunk": "^1.2.1",
     "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/disk-transform": "^1.2.1",
     "@xen-orchestra/log": "^0.7.1",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^4.5.0"
+    "xen-api": "^4.7.6"
   },
   "devDependencies": {
     "tap": "^18.7.0",

@vates/node-vsphere-soap/README.md

@@ -24,7 +24,7 @@ $ npm install node-vsphere-soap --save
 
 ## Sample Code
 
-### To connect to a vCenter server:
+### To connect to a vCenter server
 
     var nvs = require('node-vsphere-soap');
     var vc = new nvs.Client(host, user, password, sslVerify);
@@ -54,7 +54,7 @@ $ npm install node-vsphere-soap --save
 - userName - username of authenticated user
 - fullName - full name of authenticated user
 
-### To run a command:
+### To run a command
 
     var vcCmd = vc.runCommand( commandToRun, arguments );
     vcCmd.once('result', function( result, raw, soapHeader) {

@vates/node-vsphere-soap/lib/client.mjs

@@ -7,7 +7,7 @@
   This file creates the Client class
 
   - when the class is instantiated, a connection will be made to the ESXi/vCenter server to verify that the creds are good
-  - upon a bad login, the connnection will be terminated
+  - upon a bad login, the connection will be terminated
 
 */
 

@vates/node-vsphere-soap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/node-vsphere-soap",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
   "main": "lib/client.mjs",
   "author": "reedog117",
@@ -12,7 +12,7 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/log": "^0.7.1",
-    "axios": "^1.4.0",
+    "axios": "^1.8.0",
     "soap": "^1.0.0",
     "soap-cookie": "^0.10.1"
   },

@vates/otp/.USAGE.md

@@ -1,6 +1,6 @@
 ### Usual workflow
 
-> This section presents how this library should be used to implement a classic two factor authentification.
+> This section presents how this library should be used to implement a classic two factor authentication.
 
 #### Setup
 

@vates/otp/README.md

@@ -18,7 +18,7 @@ npm install --save @vates/otp
 
 ### Usual workflow
 
-> This section presents how this library should be used to implement a classic two factor authentification.
+> This section presents how this library should be used to implement a classic two factor authentication.
 
 #### Setup
 

@vates/read-chunk/index.js

@@ -71,13 +71,17 @@ exports.readChunk = readChunk
  * If `size` bytes are not available to be read, the returned promise is rejected.
  *
  * @param {Readable} stream - A readable stream to read from.
- * @param {number} [size] - The number of bytes to read for binary streams (ignored for object streams).
+ * @param {number?} [size] - The number of bytes to read for binary streams (ignored for object streams).
  * @returns {Promise<Buffer|string|unknown>} - A Promise that resolves to the read chunk.
  */
 exports.readChunkStrict = async function readChunkStrict(stream, size) {
   const chunk = await readChunk(stream, size)
   if (chunk === null) {
-    throw new Error('stream has ended without data')
+    if (size === undefined) {
+      throw new Error(`stream has ended without data`)
+    } else {
+      throw new Error(`stream has ended without data, was looking for ${size} bytes`)
+    }
   }
 
   if (size !== undefined && chunk.length !== size) {

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "1.2.0",
+  "version": "1.2.1",
   "engines": {
     "node": ">=8.10"
   },

@vates/task/.USAGE.md

@@ -63,16 +63,16 @@ task.failure(error)
 Inside a task:
 
 ```js
-// the abort signal of the current task if any, otherwise is `undefined`
+// the abort signal of the current task if any; otherwise, is `undefined`
 Task.abortSignal
 
-// sends an info on the current task if any, otherwise does nothing
+// sends an info on the current task if any; otherwise, does nothing
 Task.info(message, data)
 
-// sends an info on the current task if any, otherwise does nothing
+// sends an info on the current task if any; otherwise, does nothing
 Task.warning(message, data)
 
-// attaches a property to the current task if any, otherwise does nothing
+// attaches a property to the current task if any; otherwise, does nothing
 //
 // the latest value takes precedence
 //
@@ -149,7 +149,7 @@ const onProgress = makeOnProgress({
 
   // This function is called each time a root task or a subtask is updated.
   //
-  // `taskLog.$root` can be used to uncondionally access the root task.
+  // `taskLog.$root` can be used to unconditionally access the root task.
   onTaskUpdate(taskLog) {},
 })
 

@vates/task/README.md

@@ -79,16 +79,16 @@ task.failure(error)
 Inside a task:
 
 ```js
-// the abort signal of the current task if any, otherwise is `undefined`
+// the abort signal of the current task if any; otherwise, is `undefined`
 Task.abortSignal
 
-// sends an info on the current task if any, otherwise does nothing
+// sends an info on the current task if any; otherwise, does nothing
 Task.info(message, data)
 
-// sends an info on the current task if any, otherwise does nothing
+// sends an info on the current task if any; otherwise, does nothing
 Task.warning(message, data)
 
-// attaches a property to the current task if any, otherwise does nothing
+// attaches a property to the current task if any; otherwise, does nothing
 //
 // the latest value takes precedence
 //
@@ -165,7 +165,7 @@ const onProgress = makeOnProgress({
 
   // This function is called each time a root task or a subtask is updated.
   //
-  // `taskLog.$root` can be used to uncondionally access the root task.
+  // `taskLog.$root` can be used to unconditionally access the root task.
   onTaskUpdate(taskLog) {},
 })
 

@vates/task/index.js

@@ -29,7 +29,7 @@ class Task {
   static get abortSignal() {
     const task = getTask()
     if (task !== undefined) {
-      return task.#abortController.signal
+      return task._abortController.signal
     }
   }
 
@@ -65,8 +65,11 @@ class Task {
     }
   }
 
-  #abortController = new AbortController()
-  #onProgress
+  // These two properties should not be used outside the class
+  //
+  // To ensure compatibility between multiple versions of this lib, they are kept public for now
+  _abortController = new AbortController()
+  _onProgress
 
   get id() {
     return (this.id = Math.random().toString(36).slice(2))
@@ -86,23 +89,23 @@ class Task {
     this.#startData = { properties }
 
     if (onProgress !== undefined) {
-      this.#onProgress = onProgress
+      this._onProgress = onProgress
     } else {
       const parent = getTask()
       if (parent !== undefined) {
-        const { signal } = parent.#abortController
+        const { signal } = parent._abortController
         signal.addEventListener('abort', () => {
-          this.#abortController.abort(signal.reason)
+          this._abortController.abort(signal.reason)
         })
 
-        this.#onProgress = parent.#onProgress
+        this._onProgress = parent._onProgress
         this.#startData.parentId = parent.id
       } else {
-        this.#onProgress = noop
+        this._onProgress = noop
       }
     }
 
-    const { signal } = this.#abortController
+    const { signal } = this._abortController
     signal.addEventListener('abort', () => {
       if (this.status === PENDING) {
         this.#maybeStart()
@@ -117,7 +120,7 @@ class Task {
   }
 
   abort(reason) {
-    this.#abortController.abort(reason)
+    this._abortController.abort(reason)
   }
 
   #emit(type, data) {
@@ -126,7 +129,7 @@ class Task {
     data.id = this.id
     data.timestamp = Date.now()
     data.type = type
-    this.#onProgress(data)
+    this._onProgress(data)
   }
 
   #end(status, result) {
@@ -134,7 +137,7 @@ class Task {
     assert.equal(this.#running, false)
 
     this.#emit('end', { status, result })
-    this.#onProgress = alreadyEnded
+    this._onProgress = alreadyEnded
     this.#status = status
   }
 

@vates/task/index.test.js

@@ -29,7 +29,7 @@ function createTask(opts) {
 }
 
 describe('Task', function () {
-  describe('contructor', function () {
+  describe('constructor', function () {
     it('data properties are passed to the start event', async function () {
       const properties = { foo: 0, bar: 1 }
       const task = createTask({ properties })
@@ -151,7 +151,7 @@ describe('Task', function () {
     it('throws if the task has not started yet', function () {
       const task = createTask()
 
-      assert.throws(() => task.failure(error), { message: 'task has not started yet' })
+      assert.throws(() => task.failure(error), { message: /^task has not started yet\b/ })
     })
 
     it('throws if the task is running', function () {
@@ -262,10 +262,10 @@ describe('Task', function () {
 
     it('throws when the task is not started', function () {
       const task = createTask()
-      assert.throws(() => task.set(name, value), { message: 'task has not started yet' })
+      assert.throws(() => task.set(name, value), { message: /^task has not started yet\b/ })
     })
 
-    it(`emits an property message`, async function () {
+    it(`emits a property message`, async function () {
       const task = createTask()
       await task.run(async () => {
         await Task.run(() => {
@@ -303,7 +303,7 @@ describe('Task', function () {
     it('throws if the task has not started yet', function () {
       const task = createTask()
 
-      assert.throws(() => task.success(result), { message: 'task has not started yet' })
+      assert.throws(() => task.success(result), { message: /^task has not started yet\b/ })
     })
 
     it('throws if the task is running', function () {
@@ -339,7 +339,7 @@ describe('Task', function () {
       Task.warning('foo')
     })
 
-    it('emits an warning message when run inside a task', async function () {
+    it('emits a warning message when run inside a task', async function () {
       const task = createTask()
       await task.run(() => {
         Task.warning('foo')
@@ -356,7 +356,7 @@ describe('Task', function () {
     describe(`#${type}()`, function () {
       it('throws when the task is not started', function () {
         const task = createTask()
-        assert.throws(() => task[type]('foo'), { message: 'task has not started yet' })
+        assert.throws(() => task[type]('foo'), { message: /^task has not started yet\b/ })
       })
 
       it(`emits an ${type} message`, async function () {

@vates/task/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "engines": {
     "node": ">=14"
   },

@vates/types/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/types/eslint.config.mjs

@@ -0,0 +1,21 @@
+// @ts-check
+
+import eslint from '@eslint/js'
+import tseslint from 'typescript-eslint'
+import { fileURLToPath } from 'node:url'
+import path from 'node:path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
+  languageOptions: {
+    parserOptions: {
+      tsconfigRootDir: __dirname,
+    },
+  },
+  ignores: ['dist'],
+  rules: {
+    // Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
+    '@typescript-eslint/no-empty-object-type': 'off',
+  },
+})

@vates/types/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@vates/types",
+  "private": false,
+  "version": "1.19.0",
+  "main": "./dist/index.mjs",
+  "exports": {
+    ".": {
+      "default": "./dist/index.mjs",
+      "types": "./dist/index.d.mts"
+    },
+    "./xo": "./dist/xo.d.mts",
+    "./xo-app": "./dist/xo-app.d.mts",
+    "./xen-api": "./dist/xen-api.d.mts",
+    "./common": {
+      "default": "./dist/common.mjs",
+      "types": "./dist/common.d.mts"
+    },
+    "./lib/vates/*": {
+      "default": "./dist/lib/vates-*.mjs",
+      "type": "./dist/lib/vates-*.d.mts"
+    },
+    "./lib/xen-orchestra/*": {
+      "default": "./dist/lib/xen-orchestra-*.mjs",
+      "type": "./dist/lib/xen-orchestra-*.d.mts"
+    },
+    "./lib/complex-matcher": {
+      "default": "./dist/lib/complex-matcher.mjs",
+      "type": "./dist/lib/complex-matcher.d.mts"
+    }
+  },
+  "type": "module",
+  "devDependencies": {
+    "@eslint/js": "^9.19.0",
+    "typescript": "~5.6.3",
+    "typescript-eslint": "^8.23.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish --access public"
+  },
+  "license": "AGPL-3.0-or-later",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/types",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/types",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "engines": {
+    "node": ">=18.18"
+  }
+}

@vates/types/src/common.mts

@@ -0,0 +1,742 @@
+import type { XoHost } from './xo.mjs'
+
+declare const __brand: unique symbol
+
+export type Branded<TBrand extends string, TType = string> = TType & { [__brand]: TBrand }
+
+export const HOST_POWER_STATE = {
+  RUNNING: 'Running',
+  HALTED: 'Halted',
+  UNKNOWN: 'Unknown',
+} as const
+
+export type HOST_POWER_STATE = (typeof HOST_POWER_STATE)[keyof typeof HOST_POWER_STATE]
+
+export const TASK_ALLOWED_OPERATIONS = {
+  CANCEL: 'cancel',
+  DESTROY: 'destroy',
+} as const
+
+export type TASK_ALLOWED_OPERATIONS = (typeof TASK_ALLOWED_OPERATIONS)[keyof typeof TASK_ALLOWED_OPERATIONS]
+
+export const TASK_STATUS_TYPE = {
+  CANCELLED: 'cancelled',
+  CANCELLING: 'cancelling',
+  FAILURE: 'failure',
+  PENDING: 'pending',
+  SUCCESS: 'success',
+} as const
+
+export type TASK_STATUS_TYPE = (typeof TASK_STATUS_TYPE)[keyof typeof TASK_STATUS_TYPE]
+
+export const EVENT_OPERATION = {
+  ADD: 'add',
+  DEL: 'del',
+  MOD: 'mod',
+} as const
+
+export type EVENT_OPERATION = (typeof EVENT_OPERATION)[keyof typeof EVENT_OPERATION]
+
+export const POOL_ALLOWED_OPERATIONS = {
+  APPLY_UPDATES: 'apply_updates',
+  CERT_REFRESH: 'cert_refresh',
+  CLUSTER_CREATE: 'cluster_create',
+  CONFIGURE_REPOSITORIES: 'configure_repositories',
+  COPY_PRIMARY_HOST_CERTS: 'copy_primary_host_certs',
+  DESIGNATE_NEW_MASTER: 'designate_new_master',
+  EJECT: 'eject',
+  EXCHANGE_CA_CERTIFICATES_ON_JOIN: 'exchange_ca_certificates_on_join',
+  EXCHANGE_CERTIFICATES_ON_JOIN: 'exchange_certificates_on_join',
+  GET_UPDATES: 'get_updates',
+  HA_DISABLE: 'ha_disable',
+  HA_ENABLE: 'ha_enable',
+  SYNC_UPDATES: 'sync_updates',
+  TLS_VERIFICATION_ENABLE: 'tls_verification_enable',
+} as const
+
+export type POOL_ALLOWED_OPERATIONS = (typeof POOL_ALLOWED_OPERATIONS)[keyof typeof POOL_ALLOWED_OPERATIONS]
+
+export const TELEMETRY_FREQUENCY = {
+  DAILY: 'daily',
+  MONTHLY: 'monthly',
+  WEEKLY: 'weekly',
+} as const
+
+export type TELEMETRY_FREQUENCY = (typeof TELEMETRY_FREQUENCY)[keyof typeof TELEMETRY_FREQUENCY]
+
+export const UPDATE_SYNC_FREQUENCY = {
+  DAILY: 'daily',
+  WEEKLY: 'weekly',
+} as const
+
+export type UPDATE_SYNC_FREQUENCY = (typeof UPDATE_SYNC_FREQUENCY)[keyof typeof UPDATE_SYNC_FREQUENCY]
+
+export const AFTER_APPLY_GUIDANCE = {
+  RESTART_HOST: 'restartHost',
+  RESTART_HVM: 'restartHVM',
+  RESTART_PV: 'restartPV',
+  RESTART_XAPI: 'restartXAPI',
+} as const
+
+export type AFTER_APPLY_GUIDANCE = (typeof AFTER_APPLY_GUIDANCE)[keyof typeof AFTER_APPLY_GUIDANCE]
+
+export const UPDATE_AFTER_APPLY_GUIDANCE = {
+  RESTART_HOST: 'restartHost',
+  RESTART_HVM: 'restartHVM',
+  RESTART_PV: 'restartPV',
+  RESTART_XAPI: 'restartXAPI',
+} as const
+
+export type UPDATE_AFTER_APPLY_GUIDANCE = (typeof UPDATE_AFTER_APPLY_GUIDANCE)[keyof typeof UPDATE_AFTER_APPLY_GUIDANCE]
+
+export const LIVEPATCH_STATUS = {
+  OK: 'ok',
+  OK_LIVEPATCH_COMPLETE: 'ok_livepatch_complete',
+  OK_LIVEPATCH_INCOMPLETE: 'ok_livepatch_incomplete',
+} as const
+
+export type LIVEPATCH_STATUS = (typeof LIVEPATCH_STATUS)[keyof typeof LIVEPATCH_STATUS]
+
+export const VM_POWER_STATE = {
+  HALTED: 'Halted',
+  PAUSED: 'Paused',
+  RUNNING: 'Running',
+  SUSPENDED: 'Suspended',
+} as const
+
+export type VM_POWER_STATE = (typeof VM_POWER_STATE)[keyof typeof VM_POWER_STATE]
+
+export const UPDATE_GUIDANCES = {
+  REBOOT_HOST: 'reboot_host',
+  REBOOT_HOST_ON_KERNEL_LIVEPATCH_FAILURE: 'reboot_host_on_kernel_livepatch_failure',
+  REBOOT_HOST_ON_LIVEPATCH_FAILURE: 'reboot_host_on_livepatch_failure',
+  REBOOT_HOST_ON_XEN_LIVEPATCH_FAILURE: 'reboot_host_on_xen_livepatch_failure',
+  RESTART_DEVICE_MODEL: 'restart_device_model',
+  RESTART_TOOLSTACK: 'restart_toolstack',
+  RESTART_VM: 'restart_vm',
+} as const
+
+export type UPDATE_GUIDANCES = (typeof UPDATE_GUIDANCES)[keyof typeof UPDATE_GUIDANCES]
+
+export const ON_SOFTREBOOT_BEHAVIOR = {
+  DESTROY: 'destroy',
+  PRESERVE: 'preserve',
+  RESTART: 'restart',
+  SOFT_REBOOT: 'soft_reboot',
+} as const
+
+export type ON_SOFTREBOOT_BEHAVIOR = (typeof ON_SOFTREBOOT_BEHAVIOR)[keyof typeof ON_SOFTREBOOT_BEHAVIOR]
+
+export const ON_NORMAL_EXIT = {
+  DESTROY: 'destroy',
+  RESTART: 'restart',
+} as const
+
+export type ON_NORMAL_EXIT = (typeof ON_NORMAL_EXIT)[keyof typeof ON_NORMAL_EXIT]
+
+export const VM_OPERATIONS = {
+  ASSERT_OPERATION_VALID: 'assert_operation_valid',
+  AWAITING_MEMORY_LIVE: 'awaiting_memory_live',
+  CALL_PLUGIN: 'call_plugin',
+  CHANGING_DYNAMIC_RANGE: 'changing_dynamic_range',
+  CHANGING_MEMORY_LIMITS: 'changing_memory_limits',
+  CHANGING_MEMORY_LIVE: 'changing_memory_live',
+  CHANGING_NVRAM: 'changing_NVRAM',
+  CHANGING_SHADOW_MEMORY: 'changing_shadow_memory',
+  CHANGING_SHADOW_MEMORY_LIVE: 'changing_shadow_memory_live',
+  CHANGING_STATIC_RANGE: 'changing_static_range',
+  CHANGING_VCPUS: 'changing_VCPUs',
+  CHANGING_VCPUS_LIVE: 'changing_VCPUs_live',
+  CHECKPOINT: 'checkpoint',
+  CLEAN_REBOOT: 'clean_reboot',
+  CLEAN_SHUTDOWN: 'clean_shutdown',
+  CLONE: 'clone',
+  COPY: 'copy',
+  CREATE_TEMPLATE: 'create_template',
+  CREATE_VTPM: 'create_vtpm',
+  CSVM: 'csvm',
+  DATA_SOURCE_OP: 'data_source_op',
+  DESTROY: 'destroy',
+  EXPORT: 'export',
+  GET_BOOT_RECORD: 'get_boot_record',
+  HARD_REBOOT: 'hard_reboot',
+  HARD_SHUTDOWN: 'hard_shutdown',
+  IMPORT: 'import',
+  MAKE_INTO_TEMPLATE: 'make_into_template',
+  METADATA_EXPORT: 'metadata_export',
+  MIGRATE_SEND: 'migrate_send',
+  PAUSE: 'pause',
+  POOL_MIGRATE: 'pool_migrate',
+  POWER_STATE_RESET: 'power_state_reset',
+  PROVISION: 'provision',
+  QUERY_SERVICES: 'query_services',
+  RESUME: 'resume',
+  RESUME_ON: 'resume_on',
+  REVERT: 'revert',
+  REVERTING: 'reverting',
+  SEND_SYSRQ: 'send_sysrq',
+  SEND_TRIGGER: 'send_trigger',
+  SHUTDOWN: 'shutdown',
+  SNAPSHOT: 'snapshot',
+  SNAPSHOT_WITH_QUIESCE: 'snapshot_with_quiesce',
+  START: 'start',
+  START_ON: 'start_on',
+  SUSPEND: 'suspend',
+  UNPAUSE: 'unpause',
+  UPDATE_ALLOWED_OPERATIONS: 'update_allowed_operations',
+} as const
+
+export type VM_OPERATIONS = (typeof VM_OPERATIONS)[keyof typeof VM_OPERATIONS]
+
+export const ON_CRASH_BEHAVIOUR = {
+  COREDUMP_AND_DESTROY: 'coredump_and_destroy',
+  COREDUMP_AND_RESTART: 'coredump_and_restart',
+  DESTROY: 'destroy',
+  PRESERVE: 'preserve',
+  RENAME_RESTART: 'rename_restart',
+  RESTART: 'restart',
+} as const
+
+export type ON_CRASH_BEHAVIOUR = (typeof ON_CRASH_BEHAVIOUR)[keyof typeof ON_CRASH_BEHAVIOUR]
+
+export const DOMAIN_TYPE = {
+  HVM: 'hvm',
+  PV: 'pv',
+  PVH: 'pvh',
+  PV_IN_PVH: 'pv_in_pvh',
+  UNSPECIFIED: 'unspecified',
+} as const
+
+export type DOMAIN_TYPE = (typeof DOMAIN_TYPE)[keyof typeof DOMAIN_TYPE]
+
+export const TRISTATE_TYPE = {
+  NO: 'no',
+  UNSPECIFIED: 'unspecified',
+  YES: 'yes',
+} as const
+
+export type TRISTATE_TYPE = (typeof TRISTATE_TYPE)[keyof typeof TRISTATE_TYPE]
+
+export const VMPP_BACKUP_TYPE = {
+  CHECKPOINT: 'checkpoint',
+  SNAPSHOT: 'snapshot',
+} as const
+
+export type VMPP_BACKUP_TYPE = (typeof VMPP_BACKUP_TYPE)[keyof typeof VMPP_BACKUP_TYPE]
+
+export const VMPP_BACKUP_FREQUENCY = {
+  DAILY: 'daily',
+  HOURLY: 'hourly',
+  WEEKLY: 'weekly',
+} as const
+
+export type VMPP_BACKUP_FREQUENCY = (typeof VMPP_BACKUP_FREQUENCY)[keyof typeof VMPP_BACKUP_FREQUENCY]
+
+export const VMPP_ARCHIVE_FREQUENCY = {
+  ALWAYS_AFTER_BACKUP: 'always_after_backup',
+  DAILY: 'daily',
+  NEVER: 'never',
+  WEEKLY: 'weekly',
+} as const
+
+export type VMPP_ARCHIVE_FREQUENCY = (typeof VMPP_ARCHIVE_FREQUENCY)[keyof typeof VMPP_ARCHIVE_FREQUENCY]
+
+export const VMPP_ARCHIVE_TARGET_TYPE = {
+  CIFS: 'cifs',
+  NFS: 'nfs',
+  NONE: 'none',
+} as const
+
+export type VMPP_ARCHIVE_TARGET_TYPE = (typeof VMPP_ARCHIVE_TARGET_TYPE)[keyof typeof VMPP_ARCHIVE_TARGET_TYPE]
+
+export const VMSS_FREQUENCY = {
+  DAILY: 'daily',
+  HOURLY: 'hourly',
+  WEEKLY: 'weekly',
+} as const
+
+export type VMSS_FREQUENCY = (typeof VMSS_FREQUENCY)[keyof typeof VMSS_FREQUENCY]
+
+export const VMSS_TYPE = {
+  CHECKPOINT: 'checkpoint',
+  SNAPSHOT: 'snapshot',
+  SNAPSHOT_WITH_QUIESCE: 'snapshot_with_quiesce',
+} as const
+
+export type VMSS_TYPE = (typeof VMSS_TYPE)[keyof typeof VMSS_TYPE]
+
+export const VM_APPLIANCE_OPERATION = {
+  CLEAN_SHUTDOWN: 'clean_shutdown',
+  HARD_SHUTDOWN: 'hard_shutdown',
+  SHUTDOWN: 'shutdown',
+  START: 'start',
+} as const
+
+export type VM_APPLIANCE_OPERATION = (typeof VM_APPLIANCE_OPERATION)[keyof typeof VM_APPLIANCE_OPERATION]
+
+export const HOST_ALLOWED_OPERATIONS = {
+  APPLY_UPDATES: 'apply_updates',
+  ENABLE: 'enable',
+  EVACUATE: 'evacuate',
+  POWER_ON: 'power_on',
+  PROVISION: 'provision',
+  REBOOT: 'reboot',
+  SHUTDOWN: 'shutdown',
+  VM_MIGRATE: 'vm_migrate',
+  VM_RESUME: 'vm_resume',
+  VM_START: 'vm_start',
+} as const
+
+export type HOST_ALLOWED_OPERATIONS = (typeof HOST_ALLOWED_OPERATIONS)[keyof typeof HOST_ALLOWED_OPERATIONS]
+
+export const LATEST_SYNCED_UPDATES_APPLIED_STATE = {
+  NO: 'no',
+  UNKNOWN: 'unknown',
+  YES: 'yes',
+} as const
+
+export type LATEST_SYNCED_UPDATES_APPLIED_STATE =
+  (typeof LATEST_SYNCED_UPDATES_APPLIED_STATE)[keyof typeof LATEST_SYNCED_UPDATES_APPLIED_STATE]
+
+export const HOST_DISPLAY = {
+  DISABLED: 'disabled',
+  DISABLE_ON_REBOOT: 'disable_on_reboot',
+  ENABLED: 'enabled',
+  ENABLE_ON_REBOOT: 'enable_on_reboot',
+} as const
+
+export type HOST_DISPLAY = (typeof HOST_DISPLAY)[keyof typeof HOST_DISPLAY]
+
+export const HOST_SCHED_GRAN = {
+  CORE: 'core',
+  CPU: 'cpu',
+  SOCKET: 'socket',
+} as const
+
+export type HOST_SCHED_GRAN = (typeof HOST_SCHED_GRAN)[keyof typeof HOST_SCHED_GRAN]
+
+export const HOST_NUMA_AFFINITY_POLICY = {
+  ANY: 'any',
+  BEST_EFFORT: 'best_effort',
+  DEFAULT_POLICY: 'default_policy',
+} as const
+
+export type HOST_NUMA_AFFINITY_POLICY = (typeof HOST_NUMA_AFFINITY_POLICY)[keyof typeof HOST_NUMA_AFFINITY_POLICY]
+
+export const NETWORK_OPERATIONS = {
+  ATTACHING: 'attaching',
+} as const
+
+export type NETWORK_OPERATIONS = (typeof NETWORK_OPERATIONS)[keyof typeof NETWORK_OPERATIONS]
+
+export const NETWORK_DEFAULT_LOCKING_MODE = {
+  DISABLED: 'disabled',
+  UNLOCKED: 'unlocked',
+} as const
+
+export type NETWORK_DEFAULT_LOCKING_MODE =
+  (typeof NETWORK_DEFAULT_LOCKING_MODE)[keyof typeof NETWORK_DEFAULT_LOCKING_MODE]
+
+export const NETWORK_PURPOSE = {
+  INSECURE_NBD: 'insecure_nbd',
+  NBD: 'nbd',
+} as const
+
+export type NETWORK_PURPOSE = (typeof NETWORK_PURPOSE)[keyof typeof NETWORK_PURPOSE]
+
+export const VIF_OPERATIONS = {
+  ATTACH: 'attach',
+  PLUG: 'plug',
+  UNPLUG: 'unplug',
+} as const
+
+export type VIF_OPERATIONS = (typeof VIF_OPERATIONS)[keyof typeof VIF_OPERATIONS]
+
+export const VIF_LOCKING_MODE = {
+  DISABLED: 'disabled',
+  LOCKED: 'locked',
+  NETWORK_DEFAULT: 'network_default',
+  UNLOCKED: 'unlocked',
+} as const
+
+export type VIF_LOCKING_MODE = (typeof VIF_LOCKING_MODE)[keyof typeof VIF_LOCKING_MODE]
+
+export const VIF_IPV4_CONFIGURATION_MODE = {
+  NONE: 'None',
+  STATIC: 'Static',
+} as const
+
+export type VIF_IPV4_CONFIGURATION_MODE = (typeof VIF_IPV4_CONFIGURATION_MODE)[keyof typeof VIF_IPV4_CONFIGURATION_MODE]
+
+export const VIF_IPV6_CONFIGURATION_MODE = {
+  NONE: 'None',
+  STATIC: 'Static',
+} as const
+
+export type VIF_IPV6_CONFIGURATION_MODE = (typeof VIF_IPV6_CONFIGURATION_MODE)[keyof typeof VIF_IPV6_CONFIGURATION_MODE]
+
+export const PIF_IGMP_STATUS = {
+  DISABLED: 'disabled',
+  ENABLED: 'enabled',
+  UNKNOWN: 'unknown',
+} as const
+
+export type PIF_IGMP_STATUS = (typeof PIF_IGMP_STATUS)[keyof typeof PIF_IGMP_STATUS]
+
+export const IP_CONFIGURATION_MODE = {
+  DHCP: 'DHCP',
+  NONE: 'None',
+  STATIC: 'Static',
+} as const
+
+export type IP_CONFIGURATION_MODE = (typeof IP_CONFIGURATION_MODE)[keyof typeof IP_CONFIGURATION_MODE]
+
+export const IPV6_CONFIGURATION_MODE = {
+  AUTOCONF: 'Autoconf',
+  DHCP: 'DHCP',
+  NONE: 'None',
+  STATIC: 'Static',
+} as const
+
+export type IPV6_CONFIGURATION_MODE = (typeof IPV6_CONFIGURATION_MODE)[keyof typeof IPV6_CONFIGURATION_MODE]
+
+export const PRIMARY_ADDRESS_TYPE = {
+  IPV4: 'IPv4',
+  IPV6: 'IPv6',
+} as const
+
+export type PRIMARY_ADDRESS_TYPE = (typeof PRIMARY_ADDRESS_TYPE)[keyof typeof PRIMARY_ADDRESS_TYPE]
+
+export const BOND_MODE = {
+  ACTIVE_BACKUP: 'active-backup',
+  BALANCE_SLB: 'balance-slb',
+  LACP: 'lacp',
+} as const
+
+export type BOND_MODE = (typeof BOND_MODE)[keyof typeof BOND_MODE]
+
+export const STORAGE_OPERATIONS = {
+  DESTROY: 'destroy',
+  FORGET: 'forget',
+  PBD_CREATE: 'pbd_create',
+  PBD_DESTROY: 'pbd_destroy',
+  PLUG: 'plug',
+  SCAN: 'scan',
+  UNPLUG: 'unplug',
+  UPDATE: 'update',
+  VDI_CLONE: 'vdi_clone',
+  VDI_CREATE: 'vdi_create',
+  VDI_DATA_DESTROY: 'vdi_data_destroy',
+  VDI_DESTROY: 'vdi_destroy',
+  VDI_DISABLE_CBT: 'vdi_disable_cbt',
+  VDI_ENABLE_CBT: 'vdi_enable_cbt',
+  VDI_INTRODUCE: 'vdi_introduce',
+  VDI_LIST_CHANGED_BLOCKS: 'vdi_list_changed_blocks',
+  VDI_MIRROR: 'vdi_mirror',
+  VDI_RESIZE: 'vdi_resize',
+  VDI_SET_ON_BOOT: 'vdi_set_on_boot',
+  VDI_SNAPSHOT: 'vdi_snapshot',
+} as const
+
+export type STORAGE_OPERATIONS = (typeof STORAGE_OPERATIONS)[keyof typeof STORAGE_OPERATIONS]
+
+export const SR_HEALTH = {
+  HEALTHY: 'healthy',
+  RECOVERING: 'recovering',
+} as const
+
+export type SR_HEALTH = (typeof SR_HEALTH)[keyof typeof SR_HEALTH]
+
+export const VDI_OPERATIONS = {
+  BLOCKED: 'blocked',
+  CLONE: 'clone',
+  COPY: 'copy',
+  DATA_DESTROY: 'data_destroy',
+  DESTROY: 'destroy',
+  DISABLE_CBT: 'disable_cbt',
+  ENABLE_CBT: 'enable_cbt',
+  FORCE_UNLOCK: 'force_unlock',
+  FORGET: 'forget',
+  GENERATE_CONFIG: 'generate_config',
+  LIST_CHANGED_BLOCKS: 'list_changed_blocks',
+  MIRROR: 'mirror',
+  RESIZE: 'resize',
+  RESIZE_ONLINE: 'resize_online',
+  SET_ON_BOOT: 'set_on_boot',
+  SNAPSHOT: 'snapshot',
+  UPDATE: 'update',
+} as const
+
+export type VDI_OPERATIONS = (typeof VDI_OPERATIONS)[keyof typeof VDI_OPERATIONS]
+
+export const VDI_TYPE = {
+  CBT_METADATA: 'cbt_metadata',
+  CRASHDUMP: 'crashdump',
+  EPHEMERAL: 'ephemeral',
+  HA_STATEFILE: 'ha_statefile',
+  METADATA: 'metadata',
+  PVS_CACHE: 'pvs_cache',
+  REDO_LOG: 'redo_log',
+  RRD: 'rrd',
+  SUSPEND: 'suspend',
+  SYSTEM: 'system',
+  USER: 'user',
+} as const
+
+export type VDI_TYPE = (typeof VDI_TYPE)[keyof typeof VDI_TYPE]
+
+export const ON_BOOT = {
+  PERSIST: 'persist',
+  RESET: 'reset',
+} as const
+
+export type ON_BOOT = (typeof ON_BOOT)[keyof typeof ON_BOOT]
+
+export const VBD_OPERATIONS = {
+  ATTACH: 'attach',
+  EJECT: 'eject',
+  INSERT: 'insert',
+  PAUSE: 'pause',
+  PLUG: 'plug',
+  UNPAUSE: 'unpause',
+  UNPLUG: 'unplug',
+  UNPLUG_FORCE: 'unplug_force',
+} as const
+
+export type VBD_OPERATIONS = (typeof VBD_OPERATIONS)[keyof typeof VBD_OPERATIONS]
+
+export const VBD_TYPE = {
+  CD: 'CD',
+  DISK: 'Disk',
+  FLOPPY: 'Floppy',
+} as const
+
+export type VBD_TYPE = (typeof VBD_TYPE)[keyof typeof VBD_TYPE]
+
+export const VBD_MODE = {
+  RO: 'RO',
+  RW: 'RW',
+} as const
+
+export type VBD_MODE = (typeof VBD_MODE)[keyof typeof VBD_MODE]
+
+export const VTPM_OPERATIONS = {
+  DESTROY: 'destroy',
+} as const
+
+export type VTPM_OPERATIONS = (typeof VTPM_OPERATIONS)[keyof typeof VTPM_OPERATIONS]
+
+export const PERSISTENCE_BACKEND = {
+  XAPI: 'xapi',
+} as const
+
+export type PERSISTENCE_BACKEND = (typeof PERSISTENCE_BACKEND)[keyof typeof PERSISTENCE_BACKEND]
+
+export const CONSOLE_PROTOCOL = {
+  RDP: 'rdp',
+  RFB: 'rfb',
+  VT100: 'vt100',
+} as const
+
+export type CONSOLE_PROTOCOL = (typeof CONSOLE_PROTOCOL)[keyof typeof CONSOLE_PROTOCOL]
+
+export const CLS = {
+  CERTIFICATE: 'Certificate',
+  HOST: 'Host',
+  POOL: 'Pool',
+  PVS_PROXY: 'PVS_proxy',
+  SR: 'SR',
+  VDI: 'VDI',
+  VM: 'VM',
+  VMPP: 'VMPP',
+  VMSS: 'VMSS',
+} as const
+
+export type CLS = (typeof CLS)[keyof typeof CLS]
+
+export const TUNNEL_PROTOCOL = {
+  GRE: 'gre',
+  VXLAN: 'vxlan',
+} as const
+
+export type TUNNEL_PROTOCOL = (typeof TUNNEL_PROTOCOL)[keyof typeof TUNNEL_PROTOCOL]
+
+export const SRIOV_CONFIGURATION_MODE = {
+  MANUAL: 'manual',
+  MODPROBE: 'modprobe',
+  SYSFS: 'sysfs',
+  UNKNOWN: 'unknown',
+} as const
+
+export type SRIOV_CONFIGURATION_MODE = (typeof SRIOV_CONFIGURATION_MODE)[keyof typeof SRIOV_CONFIGURATION_MODE]
+
+export const PGPU_DOM0_ACCESS = {
+  DISABLED: 'disabled',
+  DISABLE_ON_REBOOT: 'disable_on_reboot',
+  ENABLED: 'enabled',
+  ENABLE_ON_REBOOT: 'enable_on_reboot',
+} as const
+
+export type PGPU_DOM0_ACCESS = (typeof PGPU_DOM0_ACCESS)[keyof typeof PGPU_DOM0_ACCESS]
+
+export const ALLOCATION_ALGORITHM = {
+  BREADTH_FIRST: 'breadth_first',
+  DEPTH_FIRST: 'depth_first',
+} as const
+
+export type ALLOCATION_ALGORITHM = (typeof ALLOCATION_ALGORITHM)[keyof typeof ALLOCATION_ALGORITHM]
+
+export const VGPU_TYPE_IMPLEMENTATION = {
+  GVT_G: 'gvt_g',
+  MXGPU: 'mxgpu',
+  NVIDIA: 'nvidia',
+  NVIDIA_SRIOV: 'nvidia_sriov',
+  PASSTHROUGH: 'passthrough',
+} as const
+
+export type VGPU_TYPE_IMPLEMENTATION = (typeof VGPU_TYPE_IMPLEMENTATION)[keyof typeof VGPU_TYPE_IMPLEMENTATION]
+
+export const PVS_PROXY_STATUS = {
+  CACHING: 'caching',
+  INCOMPATIBLE_PROTOCOL_VERSION: 'incompatible_protocol_version',
+  INCOMPATIBLE_WRITE_CACHE_MODE: 'incompatible_write_cache_mode',
+  INITIALISED: 'initialised',
+  STOPPED: 'stopped',
+} as const
+
+export type PVS_PROXY_STATUS = (typeof PVS_PROXY_STATUS)[keyof typeof PVS_PROXY_STATUS]
+
+export const SDN_CONTROLLER_PROTOCOL = {
+  PSSL: 'pssl',
+  SSL: 'ssl',
+} as const
+
+export type SDN_CONTROLLER_PROTOCOL = (typeof SDN_CONTROLLER_PROTOCOL)[keyof typeof SDN_CONTROLLER_PROTOCOL]
+
+export const VUSB_OPERATIONS = {
+  ATTACH: 'attach',
+  PLUG: 'plug',
+  UNPLUG: 'unplug',
+} as const
+
+export type VUSB_OPERATIONS = (typeof VUSB_OPERATIONS)[keyof typeof VUSB_OPERATIONS]
+
+export const CLUSTER_OPERATION = {
+  ADD: 'add',
+  DESTROY: 'destroy',
+  DISABLE: 'disable',
+  ENABLE: 'enable',
+  REMOVE: 'remove',
+} as const
+
+export type CLUSTER_OPERATION = (typeof CLUSTER_OPERATION)[keyof typeof CLUSTER_OPERATION]
+
+export const CLUSTER_HOST_OPERATION = {
+  DESTROY: 'destroy',
+  DISABLE: 'disable',
+  ENABLE: 'enable',
+} as const
+
+export type CLUSTER_HOST_OPERATION = (typeof CLUSTER_HOST_OPERATION)[keyof typeof CLUSTER_HOST_OPERATION]
+
+export const CERTIFICATE_TYPE = {
+  CA: 'ca',
+  HOST: 'host',
+  HOST_INTERNAL: 'host_internal',
+} as const
+
+export type CERTIFICATE_TYPE = (typeof CERTIFICATE_TYPE)[keyof typeof CERTIFICATE_TYPE]
+
+export const OPAQUE_REF = { EMPTY: 'OpaqueRef:NULL' } as const
+
+export type OPAQUE_REF_NULL = (typeof OPAQUE_REF)['EMPTY']
+
+export const BACKUP_TYPE = { backup: 'backup', metadata: 'metadataBackup', mirror: 'mirrorBackup' } as const
+
+export type BACKUP_TYPE = (typeof BACKUP_TYPE)[keyof typeof BACKUP_TYPE]
+
+export const SUPPORTED_VDI_FORMAT = {
+  raw: 'raw',
+  vhd: 'vhd',
+  qcow2: 'qcow2',
+} as const
+
+export type SUPPORTED_VDI_FORMAT = (typeof SUPPORTED_VDI_FORMAT)[keyof typeof SUPPORTED_VDI_FORMAT]
+
+// ----- XAPI Stats
+
+type XapiStatsResponse<T> = {
+  endTimestamp: number
+  interval: number
+  stats: T
+}
+type StatValues = (number | null)[]
+type RecordStatValues = Record<string, StatValues>
+
+export type XapiStatsGranularity = 'seconds' | 'minutes' | 'hours' | 'days'
+
+export type XapiHostStatsRaw = {
+  cpus?: RecordStatValues
+  ioThroughput?: {
+    r: RecordStatValues
+    w: RecordStatValues
+  }
+  iops?: {
+    r: RecordStatValues
+    w: RecordStatValues
+  }
+  iowait?: RecordStatValues
+  latency?: {
+    r: RecordStatValues
+    w: RecordStatValues
+  }
+  load?: StatValues
+  memory?: StatValues
+  memoryFree?: StatValues
+  pifs?: {
+    rx: RecordStatValues
+    tx: RecordStatValues
+  }
+}
+export type XapiHostStats = XapiStatsResponse<XapiHostStatsRaw>
+
+export type XapiVmStatsRaw = {
+  cpus?: RecordStatValues
+  cpuUsage?: StatValues
+  runstateFullrun?: StatValues
+  runstateFullContention?: StatValues
+  runstatePartialRun?: StatValues
+  runstatePartialContention?: StatValues
+  runstateConcurrencyHazard?: StatValues
+  runstateBlocked?: StatValues
+  iops?: {
+    r: RecordStatValues
+    w: RecordStatValues
+  }
+  memory?: StatValues
+  memoryFree?: StatValues
+  memoryTarget?: StatValues
+  vifs?: {
+    rx: RecordStatValues
+    tx: RecordStatValues
+  }
+  vifErrors?: {
+    rx: RecordStatValues
+    tx: RecordStatValues
+  }
+  xvds?: {
+    w?: RecordStatValues
+    r?: RecordStatValues
+    total?: RecordStatValues
+  }
+  vbdLatency?: {
+    w: RecordStatValues
+    r: RecordStatValues
+  }
+  vbdIowait?: RecordStatValues
+  vbdInflight?: RecordStatValues
+  vbdAvgquSz?: RecordStatValues
+}
+export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
+
+export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>

@vates/types/src/index.mts

@@ -0,0 +1,6 @@
+export * from './common.mjs'
+export * from './xen-api.mjs'
+export * from './xo.mjs'
+export * from './xo-app.mjs'
+export * from './lib/xen-orchestra-xapi.mjs'
+export * from './lib/complex-matcher.mjs'

@vates/types/src/lib/complex-matcher.mts

@@ -0,0 +1,5 @@
+export type Or<T> = Id<{ __or: T[] }>
+
+export type Id<T> = { id: T }
+
+export type IdOr<T> = Id<T> | Or<T>

@vates/types/src/lib/vates-task.mts

@@ -0,0 +1,21 @@
+import { Branded } from '../common.mjs'
+
+export interface VatesTask {
+  _abortController: AbortController
+  _onProgress(data?: unknown): void
+
+  id: Branded<'vates-task'>
+  status: 'failure' | 'pending' | 'success'
+
+  abort(reason?: unknown): void
+  failure(error?: unknown): void
+  info(message: string, data: unknown): void
+  run<T>(fn: () => T | (() => Promise<T>)): Promise<T>
+  runInside<T>(fn: () => T | (() => Promise<T>)): Promise<T>
+  set(name: string, value: unknown): void
+  start(): void
+  success(result?: unknown): void
+  warning(message: string, data: unknown): void
+  wrap<T>(fn: () => T | (() => Promise<T>)): Promise<T>
+  wrapInside<T>(fn: () => T | (() => Promise<T>)): Promise<T>
+}

@vates/types/src/lib/xen-orchestra-xapi.mts

@@ -0,0 +1,270 @@
+import {
+  WrappedXenApiRecord,
+  XenApiHost,
+  XenApiNetwork,
+  XenApiNetworkWrapped,
+  XenApiRecord,
+  XenApiSr,
+  XenApiTask,
+  XenApiVbd,
+  XenApiVdi,
+  XenApiVif,
+  XenApiVm,
+  XenApiVmWrapped,
+  XenApiVtpm,
+} from '../xen-api.mjs'
+import type { OPAQUE_REF_NULL, SUPPORTED_VDI_FORMAT, VBD_MODE, VBD_TYPE, VIF_LOCKING_MODE } from '../common.mjs'
+import type { PassThrough, Readable } from 'node:stream'
+import type {
+  XoGpuGroup,
+  XoHost,
+  XoNetwork,
+  XoPif,
+  XoSr,
+  XoUser,
+  XoVdi,
+  XoVgpuType,
+  XoVm,
+  XoVmTemplate,
+  XoVif,
+} from '../xo.mjs'
+
+export type XcpPatches = {
+  changelog?: {
+    author: string
+    date: number
+    description: string
+  }
+  description: string
+  license: string
+  name: string
+  release: string
+  size: number
+  url: string
+  version: string
+}
+export type XsPatches = {
+  conflicts?: string[]
+  date: string
+  description: string
+  documentationUrl?: string
+  guidances: string
+  name: string
+  id?: string
+  paid?: boolean
+  requirements?: string[]
+  upgrade?: boolean
+  url?: string
+  uuid?: string
+}
+
+export interface Xapi {
+  call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
+  callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
+
+  barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
+  getField<T extends XenApiRecord, K extends keyof T>(
+    type: Extract<WrappedXenApiRecord, T>['$type'],
+    ref: T['$ref'],
+    field: K
+  ): Promise<T[K]>
+  createNetwork(
+    params:
+      | {
+          name: string
+          description?: string
+          mtu?: number
+        }
+      | {
+          name: string
+          description?: string
+          pifId: XoPif['id']
+          mtu?: number
+          /* between 0 and 4094 */
+          vlan: number
+        }
+  ): Promise<XenApiNetworkWrapped>
+  deleteNetwork(id: XoNetwork['id']): Promise<void>
+  deleteVif(vifId: XoVif['id']): Promise<void>
+  exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
+  migrateVm(
+    vmId: XoVm['id'],
+    hostXapi: Xapi,
+    hostId: XoHost['id'],
+    opts?: {
+      force?: boolean
+      mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
+      mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
+      migrationNetworkId?: XoNetwork['id']
+      sr?: XoSr['id']
+      bypassAssert?: boolean
+    }
+  ): Promise<void>
+  listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
+  pool_emergencyShutdown(): Promise<void>
+  resumeVm(id: XoVm['id']): Promise<void>
+  unpauseVm(id: XoVm['id']): Promise<void>
+  SR_importVdi(
+    ref: XenApiSr['$ref'],
+    stream: Readable,
+    opts?: {
+      format?: SUPPORTED_VDI_FORMAT
+      name_description?: XoVdi['name_description']
+      name_label?: XoVdi['name_label']
+      other_config?: XoVdi['other_config']
+      read_only?: boolean
+      sharable?: boolean
+      SR?: XenApiSr['$ref']
+      tags?: XoVdi['tags']
+      type?: XoVdi['type']
+      virtual_size?: XoVdi['size']
+      xenstore_data?: Record<string, string>
+    }
+  ): Promise<XenApiVdi['$ref']>
+  startVm(
+    id: XoVm['id'],
+    opts?: {
+      bypassMacAddressesCheck?: boolean
+      force?: boolean
+      hostId?: XoHost['id']
+      /**
+       * if startOnly is true and the VM is not halted, throw VM_BAD_POWER_STATE
+       * otherwise, unpause/resume the VM
+       *
+       * @default false
+       */
+      startOnly?: boolean
+    }
+  ): Promise<void>
+  VM_export(
+    vmRef: XenApiVm['$ref'],
+    opts?: { cancelToken?: unknown; compress?: boolean; useSnapshot?: boolean }
+  ): ReturnType<Xapi['getResource']>
+  VM_import(
+    stream: Readable,
+    srRef?: XenApiSr['$ref'],
+    onVmCreation?: null | ((vm: XenApiVm) => unknown)
+  ): Promise<XenApiVm['$ref']>
+  createVm(
+    templateUuid: XoVmTemplate['uuid'],
+    metadataVm: {
+      affinityHost?: XoHost['id']
+      autoPoweron?: boolean
+      memory?: number
+      name_description?: string
+      name_label: string
+      nameLabel?: string
+      clone?: boolean
+      installRepository?: XoVdi['id'] | '' | null
+      vdis?: (
+        | /** Create VDI */ {
+            name_label: string
+            size: number
+            sr?: XoSr['id']
+            name_description?: string
+          }
+        | /** Update VDI*/ {
+            userdevice: string
+            name_label?: string
+            size?: number
+            sr?: XoSr['id']
+            name_description?: string
+          }
+        | /** Destroy VDI*/ { destroy: true; userdevice: string }
+      )[]
+      vifs?: (
+        | /** Create/update VIF */ {
+            device?: string
+            ipv4_allowed?: string[]
+            ipv6_allowed?: string[]
+            mac?: string
+            mtu?: number
+            network: string
+          }
+        | /** Destroy VIF */ { destroy: true; device: string }
+      )[]
+      existingVdis?: {
+        $SR: XoSr['id']
+        size: number
+        userdevice: string
+      }[]
+      vgpuType?: XoVgpuType['id']
+      gpuGroup?: XoGpuGroup['id']
+      copyHostBiosStrings?: boolean
+      hvmBootFirmware?: 'uefi' | 'bios'
+      secureBoot?: boolean
+    },
+    checkLimits?: boolean,
+    creatorId?: XoUser['id'],
+    opts?: { destroyAllVifs: boolean }
+  ): Promise<XenApiVmWrapped>
+  moveVdi(
+    vdiId: XenApiVdi['$ref'] | XoVdi['id'],
+    srId: XenApiSr['$ref'] | XoSr['id'],
+    opts?: { _failOnCbtError?: boolean }
+  ): Promise<XenApiVdi>
+  VBD_create(params: {
+    bootable?: boolean
+    empty?: boolean
+    mode?: VBD_MODE
+    other_config?: Record<string, string>
+    qos_algorithm_params?: Record<string, string>
+    qos_algorithm_type?: string
+    type?: VBD_TYPE
+    unpluggable?: boolean
+    userdevice?: string
+    VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
+    VM: XenApiVm['$ref']
+  }): Promise<XenApiVbd['$ref']>
+  VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
+  VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
+  VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
+  VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
+  VDI_exportContent(
+    vdiRef: XenApiVdi['$ref'],
+    opts: { baseRef?: string; cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
+  ): Promise<Readable & { length?: number }>
+  VDI_importContent(
+    vdiRef: XenApiVdi['$ref'],
+    stream: Readable,
+    opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
+  ): Promise<void>
+  VIF_create(
+    options: {
+      currently_attached?: boolean
+      device?: string
+      ipv4_allowed?: string[]
+      ipv6_allowed?: string[]
+      locking_mode?: VIF_LOCKING_MODE
+      MTU?: number
+      network: XenApiNetwork['$ref']
+      other_config?: Record<string, string>
+      qos_algorithm_params?: Record<string, string>
+      qos_algorithm_type?: string
+      VM: XenApiVm['$ref']
+    },
+    extraOptions?: {
+      MAC?: string
+    }
+  ): Promise<XenApiVif['$ref']>
+  VM_createCloudInitConfig(
+    vmRef: XenApiVm['$ref'],
+    cloudConfig: string,
+    opts?: { networkConfig?: string }
+  ): Promise<XoVdi['uuid']>
+  VM_destroy(
+    vmRef: XenApiVm['$ref'],
+    opts?: {
+      deleteDisks?: boolean
+      force?: boolean
+      bypassBlockedOperation?: boolean
+      forceDeleteDefaultTemplate?: boolean
+    }
+  ): Promise<void>
+  getResource(
+    pathname: string,
+    params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
+  ): Promise<{ body: Readable }>
+  isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
+  VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
+}

@vates/types/src/xo-app.mts

@@ -0,0 +1,257 @@
+import { EventEmitter } from 'node:stream'
+import type {
+  AnyXoBackupJob,
+  AnyXoJob,
+  AnyXoLog,
+  XapiXoRecord,
+  XoAuthenticationToken,
+  XoBackupRepository,
+  XoConfigBackupArchive,
+  XoGroup,
+  XoHost,
+  XoPool,
+  XoPoolBackupArchive,
+  XoProxy,
+  XoSchedule,
+  XoServer,
+  XoTask,
+  XoUser,
+  XoVm,
+  XoVmBackupArchive,
+} from './xo.mjs'
+import { VatesTask } from './lib/vates-task.mjs'
+import {
+  Xapi,
+  XapiHostStats,
+  XapiPoolStats,
+  XapiStatsGranularity,
+  XapiVmStats,
+  XenApiGpuGroupWrapped,
+  XenApiHostWrapped,
+  XenApiMessageWrapped,
+  XenApiNetworkWrapped,
+  XenApiPbdWrapped,
+  XenApiPciWrapped,
+  XenApiPgpuWrapped,
+  XenApiPifWrapped,
+  XenApiPoolWrapped,
+  XenApiSmWrapped,
+  XenApiSrWrapped,
+  XenApiVbdWrapped,
+  XenApiVdiWrapped,
+  XenApiVgpuTypeWrapped,
+  XenApiVgpuWrapped,
+  XenApiVifWrapped,
+  XenApiVmWrapped,
+  XenApiVtpmWrapped,
+} from './index.mjs'
+
+export type XapiConnection = Xapi & {
+  status: string
+  pool?: { uuid: string }
+  sessionId: string
+  _url?: { protocol: string; hostname: string; port?: string }
+}
+
+type XapiRecordByXapiXoRecord = {
+  gpuGroup: XenApiGpuGroupWrapped
+  host: XenApiHostWrapped
+  message: XenApiMessageWrapped
+  network: XenApiNetworkWrapped
+  PBD: XenApiPbdWrapped
+  PCI: XenApiPciWrapped
+  PGPU: XenApiPgpuWrapped
+  PIF: XenApiPifWrapped
+  pool: XenApiPoolWrapped
+  SR: XenApiSrWrapped
+  SM: XenApiSmWrapped
+  VBD: XenApiVbdWrapped
+  VDI: XenApiVdiWrapped
+  'VDI-snapshot': XenApiVdiWrapped
+  'VDI-unmanaged': XenApiVdiWrapped
+  vgpu: XenApiVgpuWrapped
+  vgpuType: XenApiVgpuTypeWrapped
+  VIF: XenApiVifWrapped
+  VM: XenApiVmWrapped
+  'VM-controller': XenApiVmWrapped
+  'VM-snapshot': XenApiVmWrapped
+  'VM-template': XenApiVmWrapped
+  VTPM: XenApiVtpmWrapped
+}
+export type XoApp = {
+  config: {
+    get<T = string>(path: string): T
+    getOptional(path: string): Record<string, string> | undefined
+    getOptionalDuration(path: string): number | undefined
+    getGuiRoutes(): Promise<{
+      default: {
+        url: string
+        path: string
+      }
+      v5?: {
+        url: string
+        path: string
+      }
+      v6?: {
+        url: string
+        path: string
+      }
+      [key: string]:
+        | undefined
+        | {
+            url: string
+            path: string
+          }
+    }>
+  }
+
+  objects: EventEmitter & {
+    allIndexes: {
+      type: {
+        getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
+      }
+    }
+  }
+
+  tasks: EventEmitter & {
+    abort(id: XoTask['id'], reason?: string): Promise<void>
+    clearLogs(): Promise<void>
+    create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
+    deleteLog(id: XoTask['id']): Promise<void>
+    get(id: XoTask['id']): Promise<XoTask>
+    list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
+    watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
+  }
+  apiContext: {
+    user?: XoUser
+    permission?: XoUser['permission'] | 'none' | null
+  }
+
+  // methods ------------
+  addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
+  authenticateUser: (
+    credentials: { token?: string; username?: string; password?: string },
+    userData?: { ip?: string },
+    opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
+  ) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
+  /* Throw if no authorization */
+  checkFeatureAuthorization(featureCode: string): Promise<void>
+  /* connect a server (XCP-ng/XenServer) */
+  connectXenServer(id: XoServer['id']): Promise<void>
+  createAuthenticationToken(opts: {
+    client?: {
+      id?: string
+      [key: string]: unknown
+    }
+    description?: string
+    expiresIn?: string | number
+    userId: XoUser['id']
+  }): Promise<XoAuthenticationToken>
+  createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
+  deleteGroup(id: XoGroup['id']): Promise<void>
+  deleteUser(id: XoUser['id']): Promise<void>
+  createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
+  /* disconnect a server (XCP-ng/XenServer) */
+  disconnectXenServer(id: XoServer['id']): Promise<void>
+  getAllGroups(): Promise<XoGroup[]>
+  getAllProxies(): Promise<XoProxy[]>
+  getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
+  getAllJobs(type?: string): Promise<AnyXoJob[]>
+  getProxy(id: XoProxy['id']): Promise<XoProxy>
+  getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
+  getAllRemotes(): Promise<XoBackupRepository[]>
+  getAllRemotesInfo(): Promise<
+    Record<
+      XoBackupRepository['id'],
+      {
+        size?: number
+        used?: number
+        available?: number
+        encryption?: {
+          algorithm: string
+          isLegacy: boolean
+          recommanded: string
+        }
+      }
+    >
+  >
+  getAllSchedules(): Promise<XoSchedule[]>
+  getAllUsers(): Promise<XoUser[]>
+  getAllXenServers(): Promise<XoServer[]>
+  getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
+  getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
+  getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
+  getBackupNgLogsSorted(opts: {
+    after?: number
+    before?: number
+    filter: (log: AnyXoLog) => boolean
+    limit?: number
+  }): Promise<AnyXoLog[]>
+  getGroup(id: XoGroup['id']): Promise<XoGroup>
+  getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
+  getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
+  getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
+  getObjectsByType: <T extends XapiXoRecord>(
+    type: T['type'],
+    opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
+  ) => Record<T['id'], T> | undefined
+  getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
+  getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
+  getUser: (id: XoUser['id']) => Promise<XoUser>
+  getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
+  getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
+  getXapiObject: <T extends XapiXoRecord>(
+    maybeId: T['id'] | T,
+    type: T['type'] | T['type'][]
+  ) => XapiRecordByXapiXoRecord[T['type']]
+  getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
+  getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
+  getXenServer(id: XoServer['id']): Promise<XoServer>
+  hasFeatureAuthorization(featureCode: string): Promise<boolean>
+  hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
+  listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
+    xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
+    pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
+  }>
+  listVmBackupsNg(
+    backupRepositoryIds: XoBackupRepository['id'][],
+    opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
+  ): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
+  /** Allow to add a new server in the DB (XCP-ng/XenServer) */
+  registerXenServer(
+    body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
+      allowUnauthorized?: XoServer['allowUnauthorized']
+      password: string
+      readOnly?: XoServer['readOnly']
+    }
+  ): Promise<XoServer>
+  rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise<void>
+  rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise<void>
+  removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
+  runJob(job: AnyXoJob, schedule: XoSchedule): void
+  runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
+  /** Remove a server from the DB (XCP-ng/XenServer) */
+  unregisterXenServer(id: XoServer['id']): Promise<void>
+  updateUser(
+    id: XoUser['id'],
+    updates: {
+      /**
+       * @deprecated
+       */
+      email?: string
+      authProviders?: Record<string, string>
+      name?: string
+      password?: string
+      permission?: string
+      preferences?: Record<string, string>
+    }
+  ): Promise<void>
+  updateGroup(
+    id: XoGroup['id'],
+    updates: {
+      name?: string
+    }
+  ): void
+  getAllXapis(): Record<string, XapiConnection>
+  getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
+}

@vates/types/src/xo.mts

@@ -0,0 +1,875 @@
+// Types based on xapi-object-to-xo
+
+import type {
+  Branded,
+  DOMAIN_TYPE,
+  HOST_ALLOWED_OPERATIONS,
+  HOST_POWER_STATE,
+  IP_CONFIGURATION_MODE,
+  IPV6_CONFIGURATION_MODE,
+  NETWORK_OPERATIONS,
+  PGPU_DOM0_ACCESS,
+  POOL_ALLOWED_OPERATIONS,
+  PRIMARY_ADDRESS_TYPE,
+  STORAGE_OPERATIONS,
+  VDI_OPERATIONS,
+  VDI_TYPE,
+  VIF_LOCKING_MODE,
+  VM_OPERATIONS,
+  VM_POWER_STATE,
+} from './common.mjs'
+import type * as CMType from './lib/complex-matcher.mjs'
+import type { XenApiHost, XenApiPool } from './xen-api.mjs'
+
+type BaseXapiXo = {
+  $pool: XoPool['id']
+  /**
+   * @deprecated use $pool instead
+   */
+  $poolId: XoPool['id']
+
+  _xapiRef: string
+
+  uuid: string
+}
+
+type BaseXoVm = BaseXapiXo & {
+  $VBDs: XoVbd['id'][]
+  $VGPUs: XoVgpu['id'][]
+
+  $container: XoPool['id'] | XoHost['id']
+
+  CPUs: {
+    max: number
+    number: number
+  }
+  PV_args?: string
+  VGPUs: XoVgpu['id'][]
+  VIFs: XoVif['id'][]
+  VTPMs: XoVtpm['id'][]
+
+  addresses: Record<string, string>
+  affinityHost?: XoHost['id']
+  attachedPcis?: string[]
+  auto_poweron: boolean
+  bios_strings: Record<string, string>
+  blockedOperations: Record<VM_OPERATIONS, string>
+  boot: Record<string, string>
+  coresPerSocket?: number
+  cpuCap?: number
+  cpuMask?: number[]
+  cpuWeight?: number
+  creation: Record<string, string>
+  current_operations: Record<string, VM_OPERATIONS>
+  docker?: {
+    containers?: string[]
+    enabled: boolean
+    info?: string
+    /** @deprecated */
+    process?: string
+    version?: string
+  }
+  /**
+   * @deprecated use isNestedVirtEnabled instead
+   */
+  expNestedHvm: boolean
+  isNestedVirtEnabled: boolean
+  hasVendorDevice: boolean
+  high_availability: string
+  installTime?: number | null
+  isFirmwareSupported: boolean
+  memory: {
+    dynamic: number[]
+    size: number
+    static: number[]
+    usage?: number
+  }
+  mainIpAddress?: string
+  managementAgentDetected?: boolean
+  name_description: string
+  name_label: string
+  needsVtpm: boolean
+  nicType?: string
+  notes?: string
+  os_version: Record<string, string> | null
+  other: Record<string, string>
+  parent?: XoVm['id']
+  power_state: VM_POWER_STATE
+  pvDriversDetected?: boolean
+  pvDriversUpToDate?: boolean
+  pvDriversVersion?: string
+  resourceSet?: string
+  secureBoot: boolean
+  snapshots: XoVmSnapshot['id'][]
+  startDelay: number
+  startTime?: number | null
+  suspendSr?: XoSr['id']
+  tags: string[]
+  vga?: string
+  videoram?: number
+  viridian: boolean
+  virtualizationMode: DOMAIN_TYPE
+  xenStoreData: Record<string, string>
+  /**
+   * @deprecated use pvDriversVersion instead
+   */
+  xentools?:
+    | false
+    | {
+        major: null | number
+        minor: null | number
+        version: null | string
+      }
+}
+
+export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
+  body: {
+    value?: string
+    name: string
+  }
+  object: {
+    type: XapiXoRecord['type'] | 'unknown'
+    uuid: XapiXoRecord['uuid']
+    href?: string
+  }
+}
+
+// TODO: to be typed when Bastien.N has finished working on the XO task
+type BaseXoLog = {
+  id: Branded<'xo-log'>
+  infos?: { data: unknown; message: string }[]
+  [key: string]: unknown
+}
+export type XoBackupLog = BaseXoLog & {
+  status: 'success' | 'skipped' | 'interrupted' | 'failure' | 'pending'
+  message: 'backup'
+  start: number
+  end?: number
+  tasks?: XoTask[]
+  jobId?: AnyXoBackupJob['id']
+}
+export type XoRestoreLog = BaseXoLog & {
+  message: 'restore'
+}
+export type XoVmBackupArchive = {
+  id: Branded<'vm-backup-archive'>
+  type: 'xo-vm-backup'
+  backupRepository: XoBackupRepository['id']
+  disks: { id: string; name: string; uuid: XoVdiSnapshot['uuid'] }[]
+  isImmutable?: boolean
+  jobId: XoVmBackupJob['id']
+  mode: XoVmBackupJob['mode']
+  scheduleId: XoSchedule['id']
+  size: number
+  timestamp: number
+  vm: {
+    uuid: XoVm['uuid']
+    name_description: string
+    name_label: string
+    tags: XoVm['tags']
+  }
+  differencingVhds?: number
+  dynamicVhds?: number
+  withMemory: boolean
+}
+
+type XoMetadataBackupArchive = {
+  id: Branded<'metadata-backup-archive'>
+  backupRepository: XoBackupRepository['id']
+  jobId: XoVmBackupJob['id']
+  jobName: XoVmBackupJob['name']
+  scheduleId: XoSchedule['id']
+  scheduleName: XoSchedule['name']
+  timestamp: number
+}
+export type XoConfigBackupArchive = XoMetadataBackupArchive & {
+  id: Branded<'config-backup-archive'>
+  data: string
+  type: 'xo-config-backup'
+}
+
+export type XoPoolBackupArchive = XoMetadataBackupArchive & {
+  id: Branded<'pool-backup-archive'>
+  pool: XenApiPool
+  poolMaster: XenApiHost
+  type: 'xo-pool-metadata-backup'
+}
+
+export type XoBackupRepository = {
+  benchmarks?: { readRate: number; timestamp: number; writeRate: number }[]
+  enabled: boolean
+  error?: Record<string, unknown>
+  id: Branded<'backup-repository'>
+  name: string
+  options?: string
+  proxy?: XoProxy['id']
+  url: string
+}
+
+export type XoGpuGroup = BaseXapiXo & {
+  id: Branded<'gpu-group'>
+  type: 'gpuGroup'
+}
+
+export type XoGroup = {
+  id: Branded<'group'>
+  name: string
+  provider?: string
+  providerGroupId?: string
+  users: XoUser['id'][]
+}
+
+export type XoHost = BaseXapiXo & {
+  $PBDs: XoPbd['id'][]
+  $PCIs: XoPci['id'][]
+  $PGPUs: XoPgpu['id'][]
+  $PIFs: XoPif['id'][]
+
+  PCIs: XoPci['id'][]
+  PGPUs: XoPgpu['id'][]
+  PIFs: XoPif['id'][]
+  /**
+   * @deprecated use cpus instead
+   */
+  CPUs: Record<string, string>
+
+  address: string
+  agentStartTime: null | number
+  bios_strings: Record<string, string>
+  build: string
+  certificates?: {
+    fingerprint: string
+    notAfter: number
+  }[]
+  chipset_info: {
+    iommu?: boolean
+  }
+  controlDomain?: XoVmController['id']
+  cpus: {
+    cores?: number
+    sockets?: number
+  }
+  current_operations: Record<string, HOST_ALLOWED_OPERATIONS>
+  enabled: boolean
+  hostname: string
+  hvmCapable: boolean
+  id: Branded<'host'>
+  iscsiIqn: string
+  license_expiry: null | number
+  license_params: Record<string, string>
+  license_server: Record<string, string>
+  logging: Record<string, string>
+  memory: {
+    size: number
+    /**
+     * @deprecated
+     */
+    total?: number
+    usage: number
+  }
+  multipathing: boolean
+  name_description: string
+  name_label: string
+  otherConfig: Record<string, string>
+  /**
+   * @deprecated
+   */
+  patches: XoHostPatch['id'][]
+  power_state: HOST_POWER_STATE
+  powerOnMode: string
+  productBrand: string
+  rebootRequired: boolean
+  residentVms: XoVm['id'][]
+  startTime: null | number
+  supplementalPacks:
+    | {
+        author: string
+        description: string
+        guidance: string
+        hosts: XoHost['id'][]
+        name: string
+        size: number
+        version: string
+        vdi: XoVdi['id']
+      }[]
+    | {
+        author: string
+        description: string
+        name: string
+        version: string
+      }[]
+  tags: string[]
+  type: 'host'
+  version: string
+  zstdSupported: boolean
+}
+
+export type XoHostPatch = BaseXapiXo & {
+  id: Branded<'host_patch'>
+  type: 'host_patch'
+}
+
+export type XoMessage = BaseXapiXo & {
+  $object: XapiXoRecord['id']
+
+  body: string
+  id: Branded<'message'>
+  name: string
+  time: number
+  type: 'message'
+}
+
+export type XoNetwork = BaseXapiXo & {
+  MTU: number
+  PIFs: XoPif['id'][]
+  VIFs: XoVif['id'][]
+
+  automatic: boolean
+  bridge: string
+  current_operations: Record<string, NETWORK_OPERATIONS>
+  defaultIsLocked: boolean
+  id: Branded<'network'>
+  insecureNbd?: boolean
+  name_description: string
+  name_label: string
+  nbd?: boolean
+  other_config: Record<string, string>
+  tags: string[]
+  type: 'network'
+}
+
+export type XoPbd = BaseXapiXo & {
+  attached: boolean
+  device_config:
+    | { device: string }
+    | { location: string }
+    | { path: string; location: string; legacy_mode: string }
+    | { provisioning: string; redundancy: string; 'group-name': string }
+    | { server: string; serverpath: string }
+    | { type: string; location: string }
+    | Record<string, string>
+  host: XoHost['id']
+  id: Branded<'PBD'>
+  otherConfig: Record<string, string>
+  SR: XoSr['id']
+  type: 'PBD'
+}
+
+export type XoPci = BaseXapiXo & {
+  $host?: XoHost['id']
+
+  class_name: string
+  device_name: string
+  id: Branded<'PCI'>
+  pci_id: string
+  type: 'PCI'
+}
+
+export type XoPgpu = BaseXapiXo & {
+  $host: XoHost['id']
+  $vgpus: XoVgpu['id'][]
+
+  dom0Access: PGPU_DOM0_ACCESS
+  enabledVgpuTypes: XoVgpuType['id'][]
+  gpuGroup?: XoGpuGroup['id']
+  host: XoHost['id']
+  id: Branded<'PGPU'>
+  isSystemDisplayDevice: boolean
+  pci?: XoPci['id']
+  // it seems that supportedVgpuMaxCapcities always return undefined (See: xapi-objet-to-xo)
+  /**
+   * @deprecated
+   */
+  supportedVgpuMaxCapcities?: never
+  supportedVgpuTypes: XoVgpuType['id'][]
+  type: 'PGPU'
+  vgpus: XoVgpu['id'][]
+}
+
+export type XoPif = BaseXapiXo & {
+  $host: XoHost['id']
+  $network: XoNetwork['id']
+
+  attached: boolean
+  bondMaster?: XoPif['id']
+  bondSlaves?: XoPif['id'][]
+  carrier: boolean
+  device: string
+  deviceName?: string
+  disallowUnplug: boolean
+  dns: string
+  gateway: string
+  id: Branded<'PIF'>
+  ip: string
+  ipv6: string[]
+  ipv6Mode: IPV6_CONFIGURATION_MODE
+  isBondMaster: boolean
+  isBondSlave: boolean
+  mac: string
+  management: boolean
+  mode: IP_CONFIGURATION_MODE
+  mtu: number
+  netmask: string
+  physical: boolean
+  primaryAddressType: PRIMARY_ADDRESS_TYPE
+  speed?: number
+  type: 'PIF'
+  vlan: number
+}
+
+export type XoPool = BaseXapiXo & {
+  auto_poweron: boolean
+  cpus: {
+    cores?: number
+    sockets?: number
+  }
+  crashDumpSr?: XoSr['id']
+  current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
+  default_SR?: XoSr['id']
+  HA_enabled: boolean
+  haSrs: XoSr['id'][]
+  id: Branded<'pool'>
+  master: XoHost['id']
+  migrationCompression?: boolean
+  name_description: string
+  name_label: string
+  otherConfig: Record<string, string>
+  platform_version: string
+  suspendSr?: XoSr['id']
+  tags: string[]
+  type: 'pool'
+  vtpmSupported: boolean
+  zstdSupported: boolean
+}
+
+export type XoProxy = {
+  id: Branded<'proxy'>
+  url: string
+  version?: string
+  name: string
+} & (
+  | { address?: undefined; vmUuid: XoVm['id'] }
+  | { address: string; vmUuid?: undefined }
+  | { address: string; vmUuid: XoVm['id'] }
+)
+
+type BaseXoJob = {
+  id: Branded<'job'>
+  name?: string
+}
+
+type XoBackupJobGeneralSettings = {
+  backupReportTpl?: 'compactMjml' | 'mjml'
+  reportWhen?: 'always' | 'error' | 'failure' | 'never'
+  reportRecipients?: string[]
+  hideSuccessfulItems?: boolean
+  [key: string]: unknown
+}
+
+export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
+  cbtDestroySnapshotData?: boolean
+  concurrency?: number
+  longTermRetention?: {
+    daily?: { retention: number; settings: Record<string, unknown> }
+    weekly?: { retention: number; settings: Record<string, unknown> }
+    monthly?: { retention: number; settings: Record<string, unknown> }
+    yearly?: { retention: number; settings: Record<string, unknown> }
+  }
+  checkpointSnapshot?: boolean
+  offlineSnapshot?: boolean
+  maxExportRate?: number
+  nbdConcurrency?: number
+  nRetriesVmBackupFailures?: number
+  preferNbd?: boolean
+  timezone?: string
+  mergeBackupsSynchronously?: boolean
+  offlineBackup?: boolean
+  timeout?: number
+}
+export type XoVmBackupJobScheduleSettings = {
+  exportRetention?: number
+  healthCheckVmsWithTags?: XoVm['tags']
+  fullInterval?: number
+  copyRetention?: number
+  snapshotRetention?: number
+  cbtDestroySnapshotData?: boolean
+  healthCheckSr?: XoSr['id']
+  [key: string]: unknown
+}
+export type XoVmBackupJob = BaseXoJob & {
+  compression?: 'native' | 'zstd' | ''
+  proxy?: XoProxy['id']
+  mode: 'full' | 'delta'
+  remotes?: CMType.IdOr<XoBackupRepository['id']>
+  vms: CMType.IdOr<XoVm['id']> | Record<string, unknown>
+  srs?: CMType.IdOr<XoSr['id']>
+  type: 'backup'
+  settings: {
+    '': XoVmBackupJobGeneralSettings
+    [key: XoSchedule['id']]: XoVmBackupJobScheduleSettings | undefined
+  }
+}
+
+export type XoMetadataBackupJobGeneralSettings = XoBackupJobGeneralSettings
+export type XoMetadataBackupJobScheduleSettings = {
+  retentionPoolMetadata?: number
+  retentionXoMetadata?: number
+  [key: string]: unknown
+}
+export type XoMetadataBackupJob = BaseXoJob & {
+  type: 'metadataBackup'
+  pools?: CMType.IdOr<XoPool['id']>
+  remotes: CMType.IdOr<XoBackupRepository['id']>
+  settings: {
+    ''?: XoMetadataBackupJobGeneralSettings
+    [scheduleId: XoSchedule['id']]: XoMetadataBackupJobScheduleSettings | undefined
+  }
+  xoMetadata?: boolean
+  userId: XoUser['id']
+  proxy?: XoProxy['id']
+}
+
+export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
+  concurrency?: number
+  nRetriesVmBackupFailures?: number
+  mergeBackupsSynchronously?: boolean
+  timeout?: number
+  maxExportRate?: number
+  backupReportTpl?: 'compactMjml'
+  reportWhen: 'failure'
+}
+export type XoMirrorBackupScheduleSettings = {
+  exportRetention?: number
+  healthCheckVmsWithTags?: XoVm['tags']
+  healthCheckSr?: XoSr['id']
+  [key: string]: unknown
+}
+export type XoMirrorBackupJob = BaseXoJob & {
+  type: 'mirrorBackup'
+  mode: 'full' | 'delta'
+  sourceRemote: XoBackupRepository['id']
+  remotes: CMType.IdOr<XoBackupRepository['id']>
+  proxy?: XoProxy['id']
+  settings: {
+    '': XoMirrorBackupGeneralSettings
+    [scheduleId: XoSchedule['id']]: XoMirrorBackupScheduleSettings | undefined
+  }
+}
+
+export type XoJob = BaseXoJob & {
+  type: 'call'
+}
+
+export type XoSchedule = {
+  cron: string
+  enabled: boolean
+  id: Branded<'schedule'>
+  jobId: AnyXoJob['id']
+  name?: string
+  timezone?: string
+}
+
+export type XoServer = {
+  allowUnauthorized: boolean
+  enabled: boolean
+  error?: Record<string, unknown>
+  host: string
+  httpProxy?: string
+  id: Branded<'server'>
+  label?: string
+  master?: XoHost['id']
+  poolId?: XoPool['id']
+  poolNameDescription?: string
+  poolNameLabel?: string
+  readOnly: boolean
+  status: 'connected' | 'disconnected' | 'connecting'
+  username: string
+}
+
+export type XoSr = BaseXapiXo & {
+  $PBDs: XoPbd['id'][]
+
+  $container: XoPool['id'] | XoHost['id']
+
+  VDIs: AnyXoVdi['id'][]
+
+  allocationStrategy?: 'thin' | 'thick' | 'unknown'
+  content_type: string
+  current_operations: Record<string, STORAGE_OPERATIONS>
+  id: Branded<'SR'>
+  inMaintenanceMode: boolean
+  name_description: string
+  name_label: string
+  other_config: Record<string, string>
+  physical_usage: number
+  shared: boolean
+  size: number
+  sm_config: Record<string, string>
+  SR_type: string
+  tags: string[]
+  type: 'SR'
+  usage: number
+}
+
+export type XoSm = BaseXapiXo & {
+  id: Branded<'SM'>
+
+  SM_type: string
+  vendor: string
+  name_label: string
+  name_description: string
+  configuration: Record<string, string>
+  features: Record<string, number>
+  driver_filename: string
+  required_cluster_stack: string[]
+  supported_image_formats: string[]
+  type: 'SM'
+}
+
+export type XoTask = {
+  abortionRequestedAt?: number
+  data?: Record<string, string>
+  end?: number
+  id: Branded<'task'>
+  infos?: { data: unknown; message: string }[]
+  progress?: number
+  properties: {
+    method?: string
+    name?: string
+    objectId?: string
+    objectType?:
+      | XapiXoRecord['type']
+      | 'backup'
+      | 'backup-archive'
+      | 'backup-job'
+      | 'backup-log'
+      | 'backup-repository'
+      | 'group'
+      | 'proxy'
+      | 'restore'
+      | 'restore-log'
+      | 'schedule'
+      | 'server'
+      | 'task'
+      | 'user'
+    params?: Record<string, unknown>
+    progress?: number
+    type?: string
+    userId?: string
+    [key: string]: unknown | undefined
+  }
+  result: Record<string, unknown>
+  start: number
+  status: 'failure' | 'interrupted' | 'pending' | 'success'
+  tasks?: XoTask[]
+  updatedAt?: number
+  warnings?: { data: unknown; message: string }[]
+}
+
+export type XoUser = {
+  authProviders?: Record<string, string>
+  email: string
+  groups: XoGroup['id'][]
+  id: Branded<'user'>
+  name?: string
+  permission: string
+  pw_hash?: string
+  preferences: Record<string, string>
+}
+
+export type XoAuthenticationToken = {
+  client?: {
+    id: string
+    [key: string]: unknown
+  }
+  created_at?: number
+  description?: string
+  user_id: XoUser['id']
+  expiration: number
+  last_uses?: Record<string, { timestamp: number }>
+  id: Branded<'authentication-token'>
+}
+
+export type XoVbd = BaseXapiXo & {
+  attached: boolean
+  bootable: boolean
+  device: string | null
+  id: Branded<'VBD'>
+  is_cd_drive: boolean
+  position: string
+  read_only: boolean
+  type: 'VBD'
+  VDI?: XoVdi['id'] | XoVdiSnapshot['id']
+  VM: AnyXoVm['id']
+}
+
+type BaseXoVdi = BaseXapiXo & {
+  $SR: XoSr['id']
+  $VBDs: XoVbd['id'][]
+
+  VDI_type: VDI_TYPE
+
+  cbt_enabled?: boolean
+  current_operations: Record<string, VDI_OPERATIONS>
+  missing: boolean
+  name_description: string
+  name_label: string
+  other_config: Record<string, string>
+  parent?: XoVdiUnmanaged['id']
+  image_format?: string
+  size: number
+  snapshots: XoVdiSnapshot['id'][]
+  tags: string[]
+  usage: number
+}
+
+export type XoVdi = BaseXoVdi & {
+  id: Branded<'VDI'>
+  type: 'VDI'
+}
+
+export type XoVdiSnapshot = BaseXoVdi & {
+  id: Branded<'VDI-snapshot'>
+  snapshot_time: number
+  $snapshot_of?: XoVdi['id']
+  type: 'VDI-snapshot'
+}
+
+export type XoVdiUnmanaged = Omit<BaseXoVdi, '$VBDs'> & {
+  id: Branded<'VDI-unmanaged'>
+  type: 'VDI-unmanaged'
+
+  /**
+   * Unmanaged VDI have no VBDs
+   */
+  $VBDs: never[]
+}
+
+export type XoVgpu = BaseXapiXo & {
+  id: Branded<'VGPU'>
+  type: 'vgpu'
+}
+
+export type XoVgpuType = BaseXapiXo & {
+  id: Branded<'vgpu-type'>
+  type: 'vgpuType'
+}
+
+export type XoVif = BaseXapiXo & {
+  $VM: XoVm['id']
+
+  $network: XoNetwork['id']
+
+  allowedIpv4Addresses: string[]
+  allowedIpv6Addresses: string[]
+  attached: boolean
+  device: string
+  id: Branded<'VIF'>
+  lockingMode: VIF_LOCKING_MODE
+  MAC: string
+  MTU: number
+  other_config: Record<string, string>
+  /**
+   * In kB/s
+   */
+  rateLimit?: number
+  txChecksumming: boolean
+  type: 'VIF'
+}
+
+export type XoVm = BaseXoVm & {
+  id: Branded<'VM'>
+  type: 'VM'
+  vulnerabilities: { xsa468: boolean | { reason: string; driver?: string; version?: string } }
+}
+
+export type XoVmController = BaseXoVm & {
+  id: Branded<'VM-controller'>
+  type: 'VM-controller'
+}
+
+export type XoVmSnapshot = BaseXoVm & {
+  $snapshot_of: XoVm['id']
+
+  id: Branded<'VM-snapshot'>
+  snapshot_time: number
+  suspendVdi?: XoVdi['id']
+  type: 'VM-snapshot'
+}
+
+export type XoVmTemplate = BaseXoVm & {
+  id: Branded<'VM-template'>
+  isDefaultTemplate: boolean
+  template_info: {
+    arch?: string
+    disks: {
+      bootable: true
+      size: number
+      SR: string
+      type: string
+    }[]
+    install_methods: string[]
+    install_repository?: string
+  }
+  type: 'VM-template'
+}
+
+export type XoVtpm = BaseXapiXo & {
+  id: Branded<'VTPM'>
+  type: 'VTPM'
+}
+
+export type XapiXoRecord =
+  | XoAlarm
+  | XoGpuGroup
+  | XoHost
+  | XoMessage
+  | XoNetwork
+  | XoPbd
+  | XoPci
+  | XoPgpu
+  | XoPif
+  | XoPool
+  | XoSr
+  | XoVbd
+  | XoVdi
+  | XoVdiSnapshot
+  | XoVdiUnmanaged
+  | XoVgpu
+  | XoVgpuType
+  | XoVif
+  | XoVm
+  | XoVmController
+  | XoVmSnapshot
+  | XoVmTemplate
+  | XoVtpm
+  | XoSm
+
+export type NonXapiXoRecord =
+  | AnyXoBackupArchive
+  | AnyXoJob
+  | AnyXoLog
+  | XoGroup
+  | XoProxy
+  | XoGroup
+  | XoProxy
+  | XoJob
+  | XoBackupRepository
+  | XoSchedule
+  | XoServer
+  | XoTask
+  | XoUser
+
+export type XoRecord = XapiXoRecord | NonXapiXoRecord
+
+export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
+
+export type AnyXoVdi = XoVdi | XoVdiSnapshot | XoVdiUnmanaged
+
+export type AnyXoJob = XoJob | AnyXoBackupJob
+
+export type AnyXoBackupJob = XoVmBackupJob | XoMetadataBackupJob | XoMirrorBackupJob
+
+export type AnyXoLog = XoBackupLog | XoRestoreLog
+
+export type AnyXoBackupArchive = XoVmBackupArchive | XoConfigBackupArchive | XoPoolBackupArchive

@vates/types/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist",
+    "declaration": true,
+    "strict": true,
+    "module": "ESNext",
+    "target": "ES2022",
+    "skipLibCheck": true,
+    "types": ["node"] // Only use types from Node.js and packages that are explicitly imported
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

@vates/xml/cli.js

@@ -20,7 +20,7 @@ function main([inputPath = 0]) {
   try {
     data = JSON.parse(input)
   } catch (error) {
-    // fallback to XML
+    // fall back to XML
     log(parseXml(input))
     return
   }

@xen-orchestra/audit-core/index.js

@@ -98,6 +98,17 @@ class AuditCore {
     return record
   }
 
+  async _importRecord(record) {
+    if (!record.id || !record.previousId) {
+      throw new Error('Invalid record')
+    }
+    await this._storage.put(record)
+    return {
+      id: record.id,
+      isValid: record.id === createHash(record),
+    }
+  }
+
   async checkIntegrity(oldest, newest) {
     const storage = this._storage
 

@xen-orchestra/audit-core/package.json

@@ -7,7 +7,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.3.1",
+  "version": "0.4.0",
   "engines": {
     "node": ">=14"
   },

@xen-orchestra/backups-cli/.USAGE.md

@@ -4,6 +4,10 @@ Usage:
 
     xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
 
+    alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
+
+    xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
+
       Detects and repair issues with VM backups.
 
       Options:

@xen-orchestra/backups-cli/README.md

@@ -20,6 +20,10 @@ Usage:
 
     xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
 
+    alternatively you can target the full remote, thus handling S3/Azure or encrypted remotes
+
+    xo-backups clean-vms [--merge] [--remove] --remote=REMOTEURL
+
       Detects and repair issues with VM backups.
 
       Options:

@xen-orchestra/backups-cli/commands/clean-vms.mjs

@@ -1,13 +1,81 @@
-import { asyncMap } from '@xen-orchestra/async-map'
+import { asyncEach } from '@vates/async-each'
 import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
 import { getSyncedHandler } from '@xen-orchestra/fs'
 import getopts from 'getopts'
-import { basename, dirname } from 'path'
+import { basename, dirname, join } from 'path'
 import Disposable from 'promise-toolbox/Disposable'
 import { pathToFileURL } from 'url'
 
+async function cleanFiles(vmDirs, { fix, merge, remove }) {
+  let nbSucceed = 0
+  let nbFailed = 0
+  let nbStarted = 0
+  const total = vmDirs.length
+  // don't assume all the files uses the same remote
+  await asyncEach(
+    vmDirs,
+    vmDir =>
+      Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
+        try {
+          nbStarted++
+          await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
+            fixMetadata: fix,
+            remove,
+            merge,
+            logInfo: (...args) => console.log(...args),
+            logWarn: (...args) => console.warn(...args),
+          })
+          nbSucceed++
+        } catch (error) {
+          nbFailed++
+          console.error('adapter.cleanVm', vmDir, error, { nbStarted, nbSucceed, nbFailed, total })
+        }
+      }),
+    { concurrency: 2 }
+  )
+  console.log('done', { nbSucceed, nbFailed })
+}
+
+async function cleanRemote(url, { fix, merge, remove }) {
+  let nbSucceed = 0
+  let nbFailed = 0
+  let nbStarted = 0
+  await Disposable.use(getSyncedHandler({ url }), async handler => {
+    const remoteAdapter = new RemoteAdapter(handler)
+    const vmUuids = await remoteAdapter.listAllVms()
+    const total = vmUuids.length
+    await asyncEach(
+      vmUuids,
+      async vmUuid => {
+        try {
+          nbStarted++
+          await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
+            fixMetadata: fix,
+            remove,
+            merge,
+            logInfo: (...args) => console.log(...args),
+            logWarn: (...args) => console.warn(...args),
+          })
+          nbSucceed++
+        } catch (error) {
+          nbFailed++
+          console.error('adapter.cleanVm', vmUuid, error, { nbStarted, nbSucceed, nbFailed, total })
+        }
+      },
+      { concurrency: 2 }
+    )
+  })
+  console.log('done', { nbSucceed, nbFailed })
+}
+
 export default async function cleanVms(args) {
-  const { _, fix, remove, merge } = getopts(args, {
+  const {
+    _: vmDirs,
+    fix,
+    remote,
+    remove,
+    merge,
+  } = getopts(args, {
     alias: {
       fix: 'f',
       remove: 'r',
@@ -20,19 +88,9 @@ export default async function cleanVms(args) {
     },
   })
 
-  await asyncMap(_, vmDir =>
-    Disposable.use(getSyncedHandler({ url: pathToFileURL(dirname(vmDir)).href }), async handler => {
-      try {
-        await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
-          fixMetadata: fix,
-          remove,
-          merge,
-          logInfo: (...args) => console.log(...args),
-          logWarn: (...args) => console.warn(...args),
-        })
-      } catch (error) {
-        console.error('adapter.cleanVm', vmDir, error)
-      }
-    })
-  )
+  if (remote) {
+    return cleanRemote(remote, { fix, merge, remove })
+  } else {
+    return cleanFiles(vmDirs, { fix, merge, remove })
+  }
 }

@xen-orchestra/backups-cli/package.json

@@ -6,9 +6,10 @@
   "preferGlobal": true,
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
+    "@vates/async-each": "^1.0.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.54.3",
-    "@xen-orchestra/fs": "^4.2.1",
+    "@xen-orchestra/backups": "^0.68.2",
+    "@xen-orchestra/fs": "^4.6.6",
     "filenamify": "^6.0.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +28,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.0.25",
+  "version": "1.1.7",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/.USAGE.md

@@ -0,0 +1,10 @@
+## Changing the default backup configuration
+
+This library receives a backup configuration from `xo-server` or `xo-proxy`. This configuration is split between multiple properties:
+
+- `config` contains values related to the general backup configuration
+- `config.defaultSettings`, `config.vm.defaultSettings` and `config.metadata.defaultSettings` contain values related to backup jobs default settings:
+  - `config.vm.defaultSettings` is related to VM backup jobs (see [DEFAULT_XAPI_VM_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/VmsRemote.mjs#L15))
+  - `config.metadata.defaultSettings` is related to metadata backup jobs (see [DEFAULT_METADATA_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/Metadata.mjs#L12))
+  - `config.defaultSettings` is related to all backup jobs (see [DEFAULT_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/_Abstract.mjs#L7))
+  - in case of duplicate value, `config.vm.defaultSettings` and `config.metadata.defaultSettings` will prevail over `config.defaultSettings`, and a defined job setting will prevail over any `defaultSettings`

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -4,12 +4,14 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { importIncrementalVm } from './_incrementalVm.mjs'
 import { Task } from './Task.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
-import { VhdNegative, VhdSynthetic } from 'vhd-lib'
 import { decorateClass } from '@vates/decorate-with'
 import { createLogger } from '@xen-orchestra/log'
 import { dirname, join } from 'node:path'
 import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
+import { NegativeDisk } from '@xen-orchestra/disk-transform'
+import { openDiskChain } from './disks/openDiskChain.mjs'
+import { resetVmOtherConfig } from './_otherConfig.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
 async function resolveUuid(xapi, cache, uuid, type) {
@@ -28,10 +30,10 @@ export class ImportVmBackup {
     metadata,
     srUuid,
     xapi,
-    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
+    settings: { additionalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
   }) {
     this._adapter = adapter
-    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
+    this._importIncrementalVmSettings = { additionalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
@@ -58,13 +60,13 @@ export class ImportVmBackup {
   async _reuseNearestSnapshot($defer, ignoredVdis) {
     const metadata = this._metadata
     const { mapVdisSrs } = this._importIncrementalVmSettings
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
-    const streams = {}
-    const metdataDir = dirname(metadata._filename)
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
+    const disks = {}
+    const metadataDir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
     for (const [vdiRef, vdi] of Object.entries(vdis)) {
-      const vhdPath = join(metdataDir, vhds[vdiRef])
+      const vhdPath = join(metadataDir, vhds[vdiRef])
 
       let xapiDisk
       try {
@@ -76,7 +78,7 @@ export class ImportVmBackup {
 
       let snapshotCandidate, backupCandidate
       if (xapiDisk !== undefined) {
-        debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
+        debug('found disks, will search its snapshots', { snapshots: xapiDisk.snapshots })
         for (const snapshotRef of xapiDisk.snapshots) {
           const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
 
@@ -95,7 +97,7 @@ export class ImportVmBackup {
           // have a corresponding backup more recent than metadata ?
           const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
           if (pathToSnapshotData === undefined) {
-            debug('no backup linked to this snaphot')
+            debug('no backup linked to this snapshot')
             continue
           }
           if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
@@ -110,20 +112,21 @@ export class ImportVmBackup {
         }
       }
 
-      let stream
-      const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
+      let disk
+      const backupWithSnapshotPath = join(metadataDir, backupCandidate ?? '')
       if (vhdPath === backupWithSnapshotPath) {
         // all the data are already on the host
         debug('direct reuse of a snapshot')
-        stream = null
+        disk = null
         vdis[vdiRef].baseVdi = snapshotCandidate
         // go next disk , we won't use this stream
         continue
       }
 
-      let disposableDescendants
-
-      const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
+      const parent = await openDiskChain({
+        handler: this._adapter._handler,
+        path: vhdPath,
+      })
 
       // this will also clean if another disk of this VM backup fails
       // if user really only need to restore non failing disks he can retry with ignoredVdis
@@ -132,8 +135,7 @@ export class ImportVmBackup {
         if (!disposed) {
           disposed = true
           try {
-            await disposableDescendants?.dispose()
-            await disposableSynthetic?.dispose()
+            await parent?.close()
           } catch (error) {
             warn('openVhd: failed to dispose VHDs', { error })
           }
@@ -141,11 +143,10 @@ export class ImportVmBackup {
       }
       $defer.onFailure(() => disposeOnce())
 
-      const parentVhd = disposableSynthetic.value
-      await parentVhd.readBlockAllocationTable()
-      debug('got vhd synthetic of parents', parentVhd.length)
+      debug('got vhd synthetic of parents', parent)
 
       if (snapshotCandidate !== undefined) {
+        let descendant, negativeDisk
         try {
           debug('will try to use differential restore', {
             backupWithSnapshotPath,
@@ -153,49 +154,48 @@ export class ImportVmBackup {
             vdiRef,
           })
 
-          disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
+          descendant = await openDiskChain({
+            handler: this._adapter._handler,
+            path: backupWithSnapshotPath,
             until: vhdPath,
           })
-          const descendantsVhd = disposableDescendants.value
-          await descendantsVhd.readBlockAllocationTable()
+
           debug('got vhd synthetic of descendants')
-          const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
+          negativeDisk = new NegativeDisk(parent, descendant)
           debug('got vhd negative')
 
           // update the stream with the negative vhd stream
-          stream = await negativeVhd.stream()
+          disk = negativeDisk
           vdis[vdiRef].baseVdi = snapshotCandidate
         } catch (error) {
           // can be a broken VHD chain, a vhd chain with a key backup, ....
-          // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
+          // not an irrecuperable error, don't dispose parentVhd, and fall back to full restore
           warn(`can't use differential restore`, { error })
-          disposableDescendants?.dispose()
+          descendant?.close()
+          negativeDisk?.close()
         }
       }
-      // didn't make a negative stream : fallback to classic stream
-      if (stream === undefined) {
+      // didn't make a negative stream : fall back to classic stream
+      if (disk === undefined) {
         debug('use legacy restore')
-        stream = await parentVhd.stream()
+        disk = parent
       }
-
-      stream.on('end', disposeOnce)
-      stream.on('close', disposeOnce)
-      stream.on('error', disposeOnce)
-      info('everything is ready, will transfer', stream.length)
-      streams[`${vdiRef}.vhd`] = stream
+      info('everything is ready, will transfer', disk)
+      disks[vdiRef] = disk
     }
     return {
-      streams,
+      disks,
       vbds,
       vdis,
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 
   async #decorateIncrementalVmMetadata() {
-    const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
+    const { additionalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
 
     const ignoredVdis = new Set(
       Object.entries(mapVdisSrs)
@@ -212,8 +212,8 @@ export class ImportVmBackup {
 
     const cache = new Map()
     const mapVdisSrRefs = {}
-    if (additionnalVmTag !== undefined) {
-      backup.vm.tags.push(additionnalVmTag)
+    if (additionalVmTag !== undefined) {
+      backup.vm.tags.push(additionalVmTag)
     }
     for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
       mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
@@ -240,7 +240,6 @@ export class ImportVmBackup {
       assert.strictEqual(metadata.mode, 'delta')
 
       backup = await this.#decorateIncrementalVmMetadata()
-      Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
     return Task.run(
@@ -256,7 +255,14 @@ export class ImportVmBackup {
           : await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
               newMacAddresses,
             })
-
+        const remoteName = adapter._handler._remote.name
+        let desc = `Restored on ${formatFilenameDate(+new Date())}`
+        if (remoteName !== undefined) {
+          desc += ` from ${remoteName}`
+        }
+        if (metadata.vm.name_description) {
+          desc += ` - ${metadata.vm.name_description}`
+        }
         await Promise.all([
           xapi.call('VM.add_tags', vmRef, 'restored from backup'),
           xapi.call(
@@ -264,13 +270,8 @@ export class ImportVmBackup {
             vmRef,
             `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
           ),
-          xapi.call(
-            'VM.set_name_description',
-            vmRef,
-            `Restored on ${formatFilenameDate(+new Date())} from ${adapter._handler._remote.name} -
-             ${metadata.vm.name_description}
-            `
-          ),
+          xapi.call('VM.set_name_description', vmRef, desc),
+          resetVmOtherConfig(xapi, vmRef),
         ])
 
         return {

@xen-orchestra/backups/README.md

@@ -12,6 +12,19 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backu
 npm install --save @xen-orchestra/backups
 ```
 
+## Usage
+
+## Changing the default backup configuration
+
+This library receives a backup configuration from `xo-server` or `xo-proxy`. This configuration is split between multiple properties:
+
+- `config` contains values related to the general backup configuration
+- `config.defaultSettings`, `config.vm.defaultSettings` and `config.metadata.defaultSettings` contain values related to backup jobs default settings:
+  - `config.vm.defaultSettings` is related to VM backup jobs (see [DEFAULT_XAPI_VM_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/VmsRemote.mjs#L15))
+  - `config.metadata.defaultSettings` is related to metadata backup jobs (see [DEFAULT_METADATA_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/Metadata.mjs#L12))
+  - `config.defaultSettings` is related to all backup jobs (see [DEFAULT_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/_Abstract.mjs#L7))
+  - in case of duplicate value, `config.vm.defaultSettings` and `config.metadata.defaultSettings` will prevail over `config.defaultSettings`, and a defined job setting will prevail over any `defaultSettings`
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -2,7 +2,7 @@ import { asyncEach } from '@vates/async-each'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
 import { compose } from '@vates/compose'
 import { createLogger } from '@xen-orchestra/log'
-import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
+import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { deduped } from '@vates/disposable/deduped.js'
 import { dirname, join, resolve } from 'node:path'
@@ -10,7 +10,6 @@ import { execFile } from 'child_process'
 import { mount } from '@vates/fuse-vhd'
 import { readdir, lstat } from 'node:fs/promises'
 import { synchronized } from 'decorator-synchronized'
-import { v4 as uuidv4 } from 'uuid'
 import { ZipFile } from 'yazl'
 import Disposable from 'promise-toolbox/Disposable'
 import fromCallback from 'promise-toolbox/fromCallback'
@@ -19,7 +18,7 @@ import groupBy from 'lodash/groupBy.js'
 import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
 import reduce from 'lodash/reduce.js'
-import tar from 'tar'
+import * as tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'
@@ -28,19 +27,24 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { getTmpDir } from './_getTmpDir.mjs'
 import { isMetadataFile } from './_backupType.mjs'
 import { isValidXva } from './_isValidXva.mjs'
-import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
+import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
 import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
+import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
+import { openDiskChain } from './disks/openDiskChain.mjs'
+import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
+import { ReadAhead } from '@xen-orchestra/disk-transform'
+
 export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
-const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
+const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
 
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
-const compareTimestamp = (a, b) => a.timestamp - b.timestamp
+export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 
 const noop = Function.prototype
 
@@ -201,7 +205,7 @@ export class RemoteAdapter {
     })
   }
 
-  // check if we will be allowed to merge a a vhd created in this adapter
+  // check if we will be allowed to merge a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
     return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
@@ -308,8 +312,8 @@ export class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
-    const { delta, full, ...others } = groupBy(metadatas, 'mode')
+    const metadata = await asyncMap(files, file => this.readVmBackupMetadata(file))
+    const { delta, full, ...others } = groupBy(metadata, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -423,12 +427,20 @@ export class RemoteAdapter {
   async listAllVms() {
     const handler = this._handler
     const vmsUuids = []
-    await asyncEach(await handler.list(BACKUP_DIR), async entry => {
-      // ignore hidden and lock files
-      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
-        vmsUuids.push(entry)
+    try {
+      await asyncEach(await handler.list(BACKUP_DIR), async entry => {
+        // ignore hidden and lock files
+        if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+          vmsUuids.push(entry)
+        }
+      })
+    } catch (error) {
+      // remote without any VM backup are ok
+      if (error.code !== 'ENOENT') {
+        throw error
       }
-    })
+    }
+
     return vmsUuids
   }
 
@@ -486,7 +498,7 @@ export class RemoteAdapter {
 
       const results = []
       await asyncMapSettled(partitions, partition =>
-        partition.type === LVM_PARTITION_TYPE
+        partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
           ? this._listLvmLogicalVolumes(devicePath, partition, results)
           : results.push(partition)
       )
@@ -563,7 +575,7 @@ export class RemoteAdapter {
     }
   }
 
-  async #getCachabledDataListVmBackups(dir) {
+  async #getCacheableDataListVmBackups(dir) {
     debug('generating cache', { path: dir })
 
     const handler = this._handler
@@ -601,6 +613,11 @@ export class RemoteAdapter {
   // if cache is missing  or broken  => regenerate it and return
 
   async _readCacheListVmBackups(vmUuid) {
+    // immutable remote can't use any caching
+    // since the cache file may be non modifiable
+    if (this._handler.isImmutable()) {
+      return this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
+    }
     const path = this.#getVmBackupsCache(vmUuid)
 
     const cache = await this._readCache(path)
@@ -610,7 +627,7 @@ export class RemoteAdapter {
     }
 
     // nothing cached, or cache unreadable => regenerate it
-    const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
+    const backups = await this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
     if (backups === undefined) {
       return
     }
@@ -681,26 +698,32 @@ export class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
+  async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
     const handler = this._handler
+
     if (this.useVhdDirectory()) {
-      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
-      const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
-        concurrency: writeBlockConcurrency,
-        compression: this.#getCompressionType(),
-        async validator() {
-          await input.task
-          return validator.apply(this, arguments)
+      await writeToVhdDirectory({
+        disk,
+        target: {
+          handler,
+          path,
+          concurrency: writeBlockConcurrency,
+          validator,
+          compression: 'brotli',
         },
       })
-      await VhdAbstract.createAlias(handler, path, dataPath)
-      return size
     } else {
-      return this.outputStream(path, input, { checksum, validator })
+      const stream = await toVhdStream(disk)
+      await this.outputStream(path, stream, { validator, checksum: false })
+      await validator(path)
     }
   }
 
-  async outputStream(path, input, { checksum = true, maxStreamLength, streamLength, validator = noop } = {}) {
+  async outputStream(
+    path,
+    input,
+    { checksum = !this._handler.isEncrypted, maxStreamLength, streamLength, validator = noop } = {}
+  ) {
     const container = watchStreamSize(input)
     await this._handler.outputStream(path, input, {
       checksum,
@@ -716,50 +739,37 @@ export class RemoteAdapter {
   }
 
   // open the  hierarchy of ancestors until we find a full one
-  async _createVhdStream(handler, path, { useChain }) {
-    const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
-    // I don't want the vhds to be disposed on return
-    // but only when the stream is done ( or failed )
-
-    let disposed = false
-    const disposeOnce = async () => {
-      if (!disposed) {
-        disposed = true
-        try {
-          await disposableSynthetic.dispose()
-        } catch (error) {
-          warn('openVhd: failed to dispose VHDs', { error })
-        }
-      }
+  async _createVhdDisk(handler, path, { useChain }) {
+    let disk
+    if (useChain) {
+      disk = await openDiskChain({ handler, path })
+    } else {
+      disk = new RemoteVhdDisk({ handler, path })
+      await disk.init()
     }
-    const synthetic = disposableSynthetic.value
-    await synthetic.readBlockAllocationTable()
-    const stream = await synthetic.stream()
-
-    stream.on('end', disposeOnce)
-    stream.on('close', disposeOnce)
-    stream.on('error', disposeOnce)
-    return stream
+    disk = new ReadAhead(disk)
+    return disk
   }
 
   async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
     const handler = this._handler
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const dir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
-
-    const streams = {}
+    const disks = {}
     await asyncMapSettled(Object.keys(vdis), async ref => {
-      streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
+      delete vdis[ref].baseVdi
+      disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
     })
 
     return {
-      streams,
+      disks,
       vbds,
       vdis,
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 
@@ -771,10 +781,10 @@ export class RemoteAdapter {
     let json
     let isImmutable = false
     let remoteIsImmutable = false
-    // if the remote is immutable, check if this metadatas are also immutables
+    // if the remote is immutable, check if this metadata is also immutable
     try {
       // this file is not encrypted
-      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
+      await this._handler._readFile(IMMUTABILITY_METADATA_FILENAME)
       remoteIsImmutable = true
     } catch (error) {
       if (error.code !== 'ENOENT') {
@@ -787,7 +797,7 @@ export class RemoteAdapter {
       json = await this.handler.readFile(path, { flag: 'r+' })
       // s3 handler don't respect flags
     } catch (err) {
-      // retry without triggerring immutbaility check ,only on immutable remote
+      // retry without triggering immutability check ,only on immutable remote
       if (err.code === 'EPERM' && remoteIsImmutable) {
         isImmutable = true
         json = await this._handler.readFile(path, { flag: 'r' })

@xen-orchestra/backups/RestoreMetadataBackup.mjs

@@ -24,7 +24,7 @@ export class RestoreMetadataBackup {
       const dataFileName = resolve('/', backupId, metadata.data ?? 'data.json').slice(1)
       const data = await handler.readFile(dataFileName)
 
-      // if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
+      // if data is JSON, sent it as a plain string; otherwise, consider the data as binary and encode it
       const isJson = dataFileName.endsWith('.json')
       return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
     }

@xen-orchestra/backups/_backupType.mjs

@@ -1,4 +1,5 @@
 export const isMetadataFile = filename => filename.endsWith('.json')
 export const isVhdFile = filename => filename.endsWith('.vhd')
+export const isVhdSumFile = filename => filename.endsWith('.vhd.checksum')
 export const isXvaFile = filename => filename.endsWith('.xva')
 export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -83,12 +83,12 @@ test('It remove broken vhd', async () => {
 
   await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
   assert.equal((await handler.list(basePath)).length, 1)
-  let loggued = ''
+  let logged = ''
   const logInfo = message => {
-    loggued += message
+    logged += message
   }
   await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
-  assert.equal(loggued, `VHD check error`)
+  assert.equal(logged, `VHD check error`)
   // not removed
   assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
   // really remove it
@@ -98,7 +98,7 @@ test('It remove broken vhd', async () => {
 
 test('it remove vhd with missing or multiple ancestors', async () => {
   // one with a broken parent, should be deleted
-  await generateVhd(`${basePath}/abandonned.vhd`, {
+  await generateVhd(`${basePath}/abandoned.vhd`, {
     header: {
       parentUnicodeName: 'gone.vhd',
       parentUuid: uniqueIdBuffer(),
@@ -118,21 +118,21 @@ test('it remove vhd with missing or multiple ancestors', async () => {
     `metadata.json`,
     JSON.stringify({
       mode: 'delta',
-      vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`],
+      vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
     }),
     { flags: 'w' }
   )
   // clean
-  let loggued = ''
+  let logged = ''
   const logInfo = message => {
-    loggued += message + '\n'
+    logged += message + '\n'
   }
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
 
-  const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
+  const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
   assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
 
-  // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
+  // we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
 })
 
 test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
@@ -144,12 +144,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
       vhds: [
         `${relativePath}/orphan.vhd`,
         `${relativePath}/child.vhd`,
-        // abandonned.json is not here
+        // abandoned.json is not here
       ],
     })
   )
 
-  await generateVhd(`${basePath}/abandonned.vhd`)
+  await generateVhd(`${basePath}/abandoned.vhd`)
 
   // one orphan, which is a full vhd, no parent
   const orphan = await generateVhd(`${basePath}/orphan.vhd`)
@@ -162,12 +162,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
     },
   })
 
-  let loggued = ''
+  let logged = ''
   const logInfo = message => {
-    loggued += message + '\n'
+    logged += message + '\n'
   }
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
-  let matched = loggued.match(/deleting unused VHD/g) || []
+  let matched = logged.match(/deleting unused VHD/g) || []
   assert.equal(matched.length, 1) // only one vhd should have been deleted
 
   // a missing vhd cause clean to remove all vhds
@@ -179,14 +179,14 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
         `deleted.vhd`, // in metadata but not in vhds
         `orphan.vhd`,
         `child.vhd`,
-        // abandonned.vhd is not here anymore
+        // abandoned.vhd is not here anymore
       ],
     }),
     { flags: 'w' }
   )
-  loggued = ''
+  logged = ''
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
-  matched = loggued.match(/deleting unused VHD/g) || []
+  matched = logged.match(/deleting unused VHD/g) || []
   assert.equal(matched.length, 2) // all vhds (orphan and  child  ) should have been deleted
 })
 
@@ -221,21 +221,21 @@ test('it merges delta of non destroyed chain', async () => {
     },
   })
 
-  let loggued = []
+  let logged = []
   const logInfo = message => {
-    loggued.push(message)
+    logged.push(message)
   }
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
-  assert.equal(loggued[0], `unexpected number of entries in backup cache`)
+  assert.equal(logged[0], `unexpected number of entries in backup cache`)
 
-  loggued = []
+  logged = []
   await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
-  const [merging] = loggued
+  const [merging] = logged
   assert.equal(merging, `merging VHD chain`)
 
   const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
   // size should be the size of children + grand children after the merge
-  assert.equal(metadata.size, 104960)
+  assert.equal(metadata.size, 104448)
 
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
   // only check deletion
@@ -287,9 +287,8 @@ test('it finish unterminated merge ', async () => {
 })
 
 // each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
-// the message an resulting files should be identical to the output with vhd files which is tested independantly
-
-describe('tests multiple combination ', () => {
+// the message and resulting files should be identical to the output with vhd files which is tested independently
+describe('tests multiple combination ', { concurrency: 1 }, () => {
   for (const useAlias of [true, false]) {
     for (const vhdMode of ['file', 'directory']) {
       test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
@@ -309,10 +308,10 @@ describe('tests multiple combination ', () => {
           await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
         }
 
-        // a vhd non referenced in metada
+        // a vhd non referenced in metadata
         await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
-        // an abandonded delta vhd without its parent
-        await generateVhd(`${basePath}/abandonned.vhd`, {
+        // an abandoned delta vhd without its parent
+        await generateVhd(`${basePath}/abandoned.vhd`, {
           useAlias,
           mode: vhdMode,
           header: {
@@ -403,13 +402,12 @@ describe('tests multiple combination ', () => {
 
         const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
         // size should be the size of children + grand children + clean after the merge
-        assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
+        assert.deepEqual(metadata.size, 4404224)
 
-        // broken vhd, non referenced, abandonned should be deleted ( alias and data)
+        // broken vhd, non referenced, abandoned should be deleted ( alias and data)
         // ancestor and child should be merged
         // grand child and clean vhd should not have changed
         const survivors = await handler.list(basePath)
-        // console.log(survivors)
         if (useAlias) {
           const dataSurvivors = await handler.list(basePath + '/data')
           // the goal of the alias : do not move a full folder
@@ -431,7 +429,6 @@ describe('tests multiple combination ', () => {
     }
   }
 })
-
 test('it cleans orphan merge states ', async () => {
   await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
 
@@ -456,7 +453,7 @@ test('check Aliases should work alone', async () => {
     logWarn: () => {},
   })
 
-  // only ok have suvived
+  // only ok have survived
   const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
   assert.equal(alias.length, 1)
 

@xen-orchestra/backups/_cleanVm.mjs

@@ -4,9 +4,11 @@ import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { basename, dirname, resolve } from 'node:path'
-import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
+import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
-import { mergeVhdChain } from 'vhd-lib/merge.js'
+import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
+import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
+import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
 
 import { Task } from './Task.mjs'
 import { Disposable } from 'promise-toolbox'
@@ -51,7 +53,15 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
     }
   }, 10e3)
   try {
-    return await mergeVhdChain(handler, chain, {
+    const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
+
+    const childDisks = []
+    for (const path of chain) {
+      childDisks.push(new RemoteVhdDisk({ handler, path }))
+    }
+    const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
+
+    const mergeRemoteDisk = new MergeRemoteDisk(handler, {
       logInfo,
       mergeBlockConcurrency,
       onProgress({ done: d, total: t }) {
@@ -60,6 +70,14 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
       },
       removeUnused: remove,
     })
+
+    const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
+    await parentDisk.init({ force: isResumingMerge })
+    await childDiskChain.init({ force: isResumingMerge })
+
+    const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
+
+    return result
   } finally {
     clearInterval(handle)
   }
@@ -68,10 +86,12 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
+
 const listVhds = async (handler, vmDir, logWarn) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
+  const checksums = new Set()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -85,11 +105,15 @@ const listVhds = async (handler, vmDir, logWarn) => {
         }),
         async vdiDir => {
           const list = await handler.list(vdiDir, {
-            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
 
           await asyncMap(list, async file => {
+            if (isVhdSumFile(file)) {
+              checksums.add(`${vdiDir}/${file}`)
+              return
+            }
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
@@ -111,7 +135,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
       )
   )
 
-  return { vhds, interruptedVhds, aliases }
+  return { vhds, interruptedVhds, aliases, checksums }
 }
 
 export async function checkAliases(
@@ -129,10 +153,29 @@ export async function checkAliases(
         logWarn('missing target of alias', { alias })
         if (remove) {
           logInfo('removing alias and non VHD target', { alias, target })
-          await handler.unlink(target)
           await handler.unlink(alias)
         }
+        continue
       }
+      if (err.code === 'EISDIR') {
+        logWarn('alias is a vhd directory', { alias })
+        if (remove) {
+          logInfo('removing vhd directory named as alias', { alias, target })
+          await VhdAbstract.unlink(handler, alias)
+        }
+        continue
+      }
+      logWarn('unhandled error while checking alias', { alias, err })
+      continue
+    }
+
+    if (target === '') {
+      logWarn('empty target for alias ', { alias })
+      if (remove) {
+        logInfo('removing alias and non VHD target', { alias, target })
+        await handler.unlink(alias)
+      }
+      continue
     }
 
     if (!isVhdFile(target)) {
@@ -209,7 +252,16 @@ export async function cleanVm(
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
+  const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
+
+  // from 5.110 to 5.113 we computed checksum for vhd file
+  // but never used nor removed them
+  await asyncMap(checksums, async path => {
+    if (remove) {
+      logInfo('deleting checksum file ', { path })
+      return handler.unlink(path)
+    }
+  })
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
@@ -487,14 +539,14 @@ export async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
+      const { finalDiskSize } = await limitedMergeVhdChain(handler, chain, {
         logInfo,
         logWarn,
         remove,
         mergeBlockConcurrency,
       })
-      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
-      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
+      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
+      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
     })
   }
 
@@ -575,8 +627,8 @@ export async function cleanVm(
       if (mergedSize) {
         // all disks are now key disk
         metadata.isVhdDifferencing = {}
-        for (const id of Object.values(metadata.vdis ?? {})) {
-          metadata.isVhdDifferencing[`${id}.vhd`] = false
+        for (const id of Object.keys(metadata.vdis ?? {})) {
+          metadata.isVhdDifferencing[id] = false
         }
       }
       mustRegenerateCache = true

@xen-orchestra/backups/_getOldEntries.mjs

@@ -1,4 +1,254 @@
-// returns all entries but the last retention-th
-export function getOldEntries(retention, entries) {
-  return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries
+// @ts-check
+
+import moment from 'moment-timezone'
+import assert from 'node:assert'
+/**
+ * @typedef {import('moment-timezone').Moment} Moment
+ */
+/**
+ * @typedef {Object} LTRSettings
+ * @property {number} [firstHourOfTheDay] - First hour of the day (for daily retention)
+ * @property {number} [firstDayOfWeek] - First day of the week (for weekly retention)
+ * @property {number} [firstDayOfMonth] - First day of the month (for monthly retention)
+ * @property {number} [firstDayOfYear] - First day of the year (for yearly retention)
+ */
+
+/**
+ * @typedef {Object} LTRConfig
+ * @property {number} retention - Number of time buckets to retain
+ * @property {LTRSettings} [settings] - Optional settings for the retention period
+ */
+
+/**
+ * @typedef {Object} LongTermRetention
+ * @property {LTRConfig} [daily] - Daily retention configuration
+ * @property {LTRConfig} [weekly] - Weekly retention configuration
+ * @property {LTRConfig} [monthly] - Monthly retention configuration
+ * @property {LTRConfig} [yearly] - Yearly retention configuration
+ */
+
+/**
+ * @typedef {Object} Entry
+ * @property {number | undefined} timestamp - Unix timestamp of the entry
+ * @property {string} id
+ * @property {*} [key] - Any other properties
+ */
+
+/**
+ * @typedef {Object} DateBucket
+ * @property {number} remaining - Number of remaining buckets to fill
+ * @property {string|null} lastMatchingBucket - Last bucket key that was matched
+ * @property {(date: Moment) => string} formatter - Function to format date into bucket key
+ * @property {Object.<string, Entry>} entries - Map of bucket keys to entries
+ */
+
+/**
+ * @typedef {Object} LTRDefinition
+ * @property {(options: {firstHourOfTheDay?: number, firstDayOfWeek?: number, firstDayOfMonth?: number, firstDayOfYear?: number, dateCreator: (date: Moment) => Moment}) => (date: Moment) => string} makeDateFormatter
+ * @property {string} [ancestor] - Name of the parent retention period
+ */
+
+/**
+ * Creates a function that converts dates to moment objects in a specific timezone
+ * @param {string|undefined} timezone - IANA timezone string (e.g., 'America/New_York')
+ * @returns {(date: Moment | number) => Moment} Function that creates timezoned moment objects
+ */
+
+function instantiateTimezonedDateCreator(timezone) {
+  return date => {
+    const transformed = timezone ? moment.tz(date, timezone) : moment(date)
+    assert.ok(transformed.isValid(), `date ${date} , timezone ${timezone} is invalid`)
+    return transformed
+  }
+}
+
+/**
+ * @type {Object.<string, LTRDefinition>}
+ */
+const LTR_DEFINITIONS = {
+  daily: {
+    makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator }) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.hour(copy.hour() - firstHourOfTheDay)
+        return copy.format('YYYY-MM-DD')
+      }
+    },
+  },
+  weekly: {
+    makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator }) => {
+      return date => {
+        const copy = dateCreator(date)
+
+        copy.date(copy.date() - firstDayOfWeek)
+        // warning, the year in term of week may different from YYYY
+        // since the computation of the first week of a year is timezone dependent
+        return copy.format('gggg-ww')
+      }
+    },
+    ancestor: 'daily',
+  },
+  monthly: {
+    makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator }) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfMonth)
+        return copy.format('YYYY-MM')
+      }
+    },
+    ancestor: 'weekly',
+  },
+  yearly: {
+    makeDateFormatter: ({ firstDayOfYear = 0, dateCreator }) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfYear)
+        return copy.format('YYYY')
+      }
+    },
+    ancestor: 'monthly',
+  },
+}
+/**
+ * Groups entries into date buckets based on long-term retention configuration
+ * @template {Entry} T
+ * @param {Array<T>} entries - Array of entries sorted in descending order by timestamp
+ * @param {LongTermRetention} longTermRetention - Configuration for retention periods
+ * @param {string | undefined} timezone - IANA timezone string
+ * @returns {Object.<string, DateBucket>} Map of retention period names to their date buckets
+ */
+
+export function getLtrEntries(entries, longTermRetention, timezone) {
+  /**
+   * @type  {Object.<string, DateBucket>}
+   */
+  const dateBuckets = {}
+  if (!longTermRetention || Object.keys(longTermRetention).length === 0) {
+    return {}
+  }
+  const dateCreator = instantiateTimezonedDateCreator(timezone)
+  // only check buckets that have a retention set
+  for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
+    assert.notStrictEqual(LTR_DEFINITIONS[duration], undefined, `Retention of type ${duration} is not defined`)
+    assert.notStrictEqual(
+      timezone,
+      undefined,
+      `timezone must defined for ltr, got ${JSON.stringify({ longTermRetention, timezone })}`
+    )
+    assert.strictEqual(
+      typeof retention,
+      'number',
+      `retention  of type ${duration} must be a number, got ${JSON.stringify(retention)}`
+    )
+    assert.strictEqual(
+      retention > 0,
+      true,
+      `retention  of type ${duration} must be a positive number, got ${JSON.stringify(retention)}`
+    )
+    dateBuckets[duration] = {
+      remaining: retention,
+      lastMatchingBucket: null,
+      formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
+      entries: {}, // bucket => entry id
+    }
+  }
+  const nb = entries.length
+  let previousTimestamp = -1
+  for (let i = nb - 1; i >= 0; i--) {
+    const entry = entries[i]
+    // force it so the compiler is sure we have a timestamp
+    if (entry.timestamp === undefined) {
+      assert.notStrictEqual(
+        entry.timestamp,
+        undefined,
+        `Can't compute long term retention if entries don't have a timestamp`
+      )
+      continue
+    }
+    // we go through the entries from the last (most recent) to the first (oldest)
+    assert.ok(
+      previousTimestamp === -1 || entry.timestamp < previousTimestamp,
+      `entries must be sorted in desc order ${new Date(entry.timestamp)} , previous :  > ${new Date(previousTimestamp)} `
+    )
+    previousTimestamp = entry.timestamp
+    const entryDate = dateCreator(entry.timestamp)
+    for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
+      const bucket = formatter(entryDate)
+      if (lastMatchingBucket !== bucket) {
+        if (remaining === 0) {
+          continue
+        }
+        dateBuckets[duration].lastMatchingBucket = bucket
+        dateBuckets[duration].remaining -= 1
+      }
+      dateBuckets[duration].entries[bucket] = entry
+    }
+  }
+
+  return dateBuckets
+}
+
+/**
+ * return the status of an entry
+ * @param {Array<Entry>} entries
+ * @param {Entry} entry
+ * @param {LongTermRetention} longTermRetention - Configuration for retention periods
+ * @param {string | undefined} timezone - IANA timezone string
+ * @returns {Array<{duration: string, dateBucket:string}>}
+ */
+export function getEntryStatus(entries, entry, longTermRetention, timezone) {
+  const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
+  const entryBuckets = []
+  for (const [duration, { entries }] of Object.entries(dateBuckets)) {
+    for (const [dateBucket, bucketEntry] of Object.entries(entries)) {
+      if (entry.id === bucketEntry.id) {
+        entryBuckets.push({ duration, dateBucket })
+      }
+    }
+  }
+  return entryBuckets
+}
+
+/**
+ * return the entries too old to be kept
+ *  if multiple entries are in the same time bucket: keep only the oldest one
+ *  if an entry is valid in any of the bucket OR the minRetentionCount: keep it
+ *  if a bucket is completely empty: it does not count as one, thus it may extend the retention
+ * if an entry is within the most recents minRetentionCount entries : keep it
+ *
+ * @param {number} minRetentionCount - Minimum number of most recent entries to always keep
+ * @param {Array<Entry>} entries - Array of entries sorted in descending order by timestamp
+ * @param {Object} options - Additional options
+ * @param {LongTermRetention} [options.longTermRetention={}] - Configuration for retention periods
+ * @param {string | undefined} [options.timezone] - IANA timezone string
+ * @returns {Array<Entry>} Array of entries that can be removed
+ */
+export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
+  assert.strictEqual(
+    typeof minRetentionCount,
+    'number',
+    `minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  assert.ok(
+    minRetentionCount >= 0,
+    `minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
+
+  const kept = new Set(entries.filter((_, index) => index >= entries.length - minRetentionCount))
+  /**
+   * @type {Set<Entry>}
+   */
+  for (const { entries } of Object.values(dateBuckets)) {
+    for (const entry of Object.values(entries)) {
+      kept.add(entry)
+    }
+  }
+
+  // ensure order is the same as the source
+  const oldEntries = entries.filter((entry, index) => {
+    return !kept.has(entry)
+  })
+
+  return oldEntries
 }

@xen-orchestra/backups/_getOldEntries.test.mjs

@@ -0,0 +1,306 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { getOldEntries } from './_getOldEntries.mjs'
+
+describe('_getOldEntries() should succeed', () => {
+  const tests = [
+    {
+      args: [
+        1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 2, id: 2 },
+          { timestamp: 3, id: 3 },
+        ],
+      ],
+      expectedIds: [1, 2],
+      testLabel: 'should  handle number based retention ',
+    },
+    {
+      args: [3, [{ id: 1 }, 4, new Date(), 'whatever']],
+      expectedIds: [1],
+      testLabel: 'should  handle number based retention without timestamp  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-09-01 00:00:00'), id: 1 }, // too old
+          { timestamp: +new Date('2024-09-01 00:01:00'), id: 2 }, // too old
+          { timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // oldest in same day
+          { timestamp: +new Date('2024-09-02 00:10:00'), id: 4 },
+          { timestamp: +new Date('2024-09-03 00:09:00'), id: 5 },
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // oldest in same day
+          { timestamp: +new Date('2024-09-04 00:10:00'), id: 7 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 2, 4, 7],
+      testLabel: 'should  handle day based retention  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-08-30 00:01:00'), id: 1 }, // week n-3 too old
+          { timestamp: +new Date('2024-08-31 00:00:00'), id: 2 }, // week n-3 too old
+          { timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2, oldest
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 4 }, // week n-2
+          { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // week n-2
+          { timestamp: +new Date('2024-09-06 00:09:00'), id: 6 }, // week n-2
+          { timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2
+          { timestamp: +new Date('2024-09-15 00:09:00'), id: 8 }, // week n-1, oldest
+          { timestamp: +new Date('2024-09-21 00:09:00'), id: 9 }, // week n-1  most recent
+          { timestamp: +new Date('2024-09-22 00:09:00'), id: 10 }, // week n  kept
+        ],
+        {
+          longTermRetention: {
+            weekly: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 2, 4, 5, 6, 7, 9],
+      testLabel: 'should  handle week based retention  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-06-22 00:09:00'), id: 1 }, // too old
+          { timestamp: +new Date('2024-07-31 00:09:00'), id: 2 }, // first of july
+          { timestamp: +new Date('2024-08-01 00:09:00'), id: 3 }, // older of august
+          { timestamp: +new Date('2024-08-05 00:09:00'), id: 4 }, // older of august
+          { timestamp: +new Date('2024-08-07 00:09:00'), id: 5 }, // most recent of august
+          { timestamp: +new Date('2024-09-09 00:09:00'), id: 6 }, // older of september
+          { timestamp: +new Date('2024-09-15 00:09:00'), id: 7 }, // older of september
+          { timestamp: +new Date('2024-09-22 00:09:00'), id: 8 }, // most recent of september
+        ],
+        {
+          longTermRetention: {
+            monthly: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 4, 5, 7, 8],
+      testLabel: 'should  handle month based retention',
+    },
+    {
+      args: [
+        2,
+        [
+          { timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, //
+          { timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, //  year n -1
+          { timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, //
+          { timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, //
+          { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, //
+          { timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, //  month n-4
+          { timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, //  month n-3
+          { timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, //  week n-3
+          { timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, //  week n-2, month n-2,
+          { timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, //  week n-1, year n
+          { timestamp: +new Date('2025-01-02 00:10:00'), id: 11 }, //  day n-1, month n-1,
+          { timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, //  day n , week n , month n
+          { timestamp: +new Date('2025-12-31 10:09:00'), id: 13 }, // min retention
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 2 },
+            weekly: { retention: 4 },
+            monthly: { retention: 5 },
+            yearly: { retention: 2 },
+          },
+          timezone: 'Europe/Paris', // use a time zone here because week definition is timezone bound
+        },
+      ],
+      expectedIds: [1, 3, 4, 5],
+      testLabel: 'complete test  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-08-31 09:01:00'), id: 1 }, // 4th day too old
+          { timestamp: +new Date('2024-09-01 00:10:00'), id: 2 }, // third day
+          { timestamp: +new Date('2024-09-01 14:20:00'), id: 3 }, // second day, oldest
+          { timestamp: +new Date('2024-09-02 00:22:00'), id: 4 }, // second day
+          { timestamp: +new Date('2024-09-03 14:00:00'), id: 5 }, // same day oldest
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // same day in NZ
+          { timestamp: +new Date('2024-09-04 00:10:00'), id: 7 }, // most recent
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 3 },
+          },
+          timezone: 'Pacific/Auckland', // GMT +13
+        },
+      ],
+      expectedIds: [1, 4, 6, 7],
+      testLabel: 'should  handle timezone ',
+    },
+  ]
+
+  for (const { args, expectedIds, testLabel } of tests) {
+    it(testLabel, () => {
+      const oldEntries = getOldEntries.apply(null, args)
+      assert.strictEqual(
+        oldEntries.length,
+        expectedIds.length,
+        `different length , ${JSON.stringify({ expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
+      )
+      for (let i = 0; i < expectedIds.length; i++) {
+        if (i > 0) {
+          assert.strictEqual(
+            expectedIds[i - 1] < expectedIds[i],
+            true,
+            `expect ids in ascending order ${expectedIds[i - 1]} ${expectedIds[i]}`
+          )
+        }
+        assert.strictEqual(
+          oldEntries[i].id,
+          expectedIds[i],
+          `different id , ${JSON.stringify({ i, expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
+        )
+      }
+    })
+  }
+})
+
+describe('_getOldEntries() should fail when called incorrectly', () => {
+  const tests = [
+    {
+      args: [
+        -1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+      ],
+      testLabel: 'negative min retention ',
+    },
+    {
+      args: [
+        { not: 'a number' },
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+      ],
+      testLabel: 'non number min retention ',
+    },
+    {
+      args: [
+        4,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 2, id: 2 },
+          { timestamp: 3, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            invalidType: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'invalid type ltr ',
+    },
+    {
+      args: [
+        3,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 0 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'zero retention ltr ',
+    },
+    {
+      args: [
+        1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 2, id: 2 },
+          { timestamp: 3, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 'not a number' },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'no timezone ltr ',
+    },
+
+    {
+      args: [
+        5,
+        [
+          { timestamp: +new Date('2024-09-01'), id: 1 },
+          { timestamp: +new Date('2024-09-03'), id: 3 },
+          { timestamp: +new Date('2024-09-02'), id: 2 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 5 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'unsorted entries ',
+    },
+
+    {
+      args: [
+        5,
+        [
+          { timestamp: 'notadate', id: 1 },
+          { timestamp: 'still not a date, but after', id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 5 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'broken date ',
+    },
+    {
+      args: [
+        1,
+        [{ obj: 1 }, 'toto', Date.now()],
+        {
+          longTermRetention: {
+            daily: { retention: 5 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'should fail if LTR without timestamp',
+    },
+  ]
+
+  for (const { args, testLabel } of tests) {
+    it(testLabel, () => {
+      assert.throws(() => getOldEntries.apply(null, args), { code: 'ERR_ASSERTION' })
+    })
+  }
+})

@xen-orchestra/backups/_incrementalVm.mjs

@@ -10,6 +10,10 @@ import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
 import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
 
+import { VHD_MAX_SIZE, XapiDiskSource } from '@xen-orchestra/xapi'
+import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
+import { toQcow2Stream } from '@xen-orchestra/qcow2'
+
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
 
 export async function exportIncrementalVm(
@@ -19,7 +23,7 @@ export async function exportIncrementalVm(
 ) {
   // refs of VM's VDIs → base's VDIs.
 
-  const streams = {}
+  const disks = {}
   const vdis = {}
   const vbds = {}
   await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
@@ -52,32 +56,14 @@ export async function exportIncrementalVm(
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
     }
-    try {
-      streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-        baseRef: baseVdi?.$ref,
-        cancelToken,
-        format: 'vhd',
-        nbdConcurrency,
-        preferNbd,
-      })
-    } catch (err) {
-      if (err.code === 'VDI_CANT_DO_DELTA') {
-        // fall back to a base
-        Task.info(`Can't do delta, will try to get a full stream`, { vdi })
-        streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-          cancelToken,
-          format: 'vhd',
-          nbdConcurrency,
-          preferNbd,
-        })
-        // only warn if the fall back succeed
-        Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
-          vdi,
-        })
-      } else {
-        throw err
-      }
-    }
+    disks[vdiRef] = new XapiDiskSource({
+      vdiRef,
+      xapi: vm.$xapi,
+      baseRef: baseVdi?.$ref,
+      nbdConcurrency,
+      preferNbd,
+    })
+    await disks[vdiRef].init()
   })
 
   const suspendVdi = vm.$suspend_VDI
@@ -87,10 +73,13 @@ export async function exportIncrementalVm(
       ...suspendVdi,
       $SR$uuid: suspendVdi.$SR.uuid,
     }
-    streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
-      cancelToken,
-      format: 'vhd',
+    disks[vdiRef] = new XapiDiskSource({
+      vdiRef: suspendVdi.$ref,
+      xapi: vm.$xapi,
+      nbdConcurrency,
+      preferNbd,
     })
+    await disks[vdiRef].init()
   }
 
   const vifs = {}
@@ -104,23 +93,31 @@ export async function exportIncrementalVm(
     }
   })
 
-  return Object.defineProperty(
-    {
-      version: '1.1.0',
-      vbds,
-      vdis,
-      vifs,
-      vm: {
-        ...vm,
-      },
-    },
-    'streams',
-    {
-      configurable: true,
-      value: streams,
-      writable: true,
-    }
+  // Get a fresh list of VM's VTPM to avoid `vm.VTPMs: [undefined]`
+  const vmVtpms = await vm.$xapi.getField('VM', vm.$ref, 'VTPMs')
+  const vtpms = await Promise.all(
+    vmVtpms.map(async vtpmRef => {
+      let content
+      try {
+        content = await vm.$xapi.call('VTPM.get_contents', vtpmRef)
+      } catch (err) {
+        console.error(err)
+      }
+      return content
+    })
   )
+
+  return {
+    version: '1.1.0',
+    vbds,
+    vdis,
+    vifs,
+    vm: {
+      ...vm,
+    },
+    vtpms,
+    disks,
+  }
 }
 
 export const importIncrementalVm = defer(async function importIncrementalVm(
@@ -186,8 +183,8 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     const vdi = vdiRecords[vdiRef]
     let newVdi
 
-    if (vdi.baseVdi !== undefined) {
-      newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
+    if (vdi.baseVdi?.$ref !== undefined) {
+      newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
       $defer.onFailure(() => newVdi.$destroy())
 
       await newVdi.update_other_config(COPY_OF, vdi.uuid)
@@ -231,21 +228,28 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     }
   })
 
-  const { streams } = incrementalVm
-
+  const { disks } = incrementalVm
   await Promise.all([
     // Import VDI contents.
     cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
-      for (let stream of ensureArray(streams[`${id}.vhd`])) {
-        if (stream === null) {
-          // we restore a backup and reuse completly a local snapshot
+      for (const disk of ensureArray(disks[id])) {
+        if (disk === null) {
+          // we restore a backup and reuse completely a local snapshot
           continue
         }
-        if (typeof stream === 'function') {
-          stream = await stream()
-        }
         await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
-        await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
+
+        let stream, format
+        if (vdi.virtual_size > VHD_MAX_SIZE) {
+          stream = await toQcow2Stream(disk)
+          format = 'qcow2'
+        } else {
+          stream = await toVhdStream(disk)
+          format = 'vhd'
+        }
+
+        await vdi.$importContent(stream, { cancelToken, format })
+
         await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
       }
     }),
@@ -281,10 +285,22 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       }
     }),
   ])
-
+  // recreate VTPMs
+  await Promise.all(
+    (incrementalVm.vtpms ?? []).map(async contents => {
+      await xapi.VTPM_create({ VM: vmRef, contents })
+    })
+  )
+  const vm = await xapi.getRecord('VM', vmRef)
   await Promise.all([
-    incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
-    xapi.setField('VM', vmRef, 'name_label', incrementalVm.vm.name_label),
+    vmRecord.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
+    xapi.setField('VM', vmRef, 'name_label', vmRecord.name_label),
+    // correctly unlock the VM and reapply the target blocked operations
+    vm.update_blocked_operations({
+      start: null,
+      start_on: null,
+      ...vmRecord.blocked_operations,
+    }),
   ])
 
   return vmRef

@xen-orchestra/backups/_isValidXva.mjs

@@ -44,7 +44,7 @@ const isValidTar = async (handler, size, fd) => {
   return buf.every(_ => _ === 0)
 }
 
-// TODO: find an heuristic for compressed files
+// TODO: find a heuristic for compressed files
 export async function isValidXva(path) {
   const handler = this._handler
 

@xen-orchestra/backups/_listPartitions.mjs

@@ -20,13 +20,31 @@ const IGNORED_PARTITION_TYPES = {
   0xd5: true,
 
   0x82: true, // swap
+  // GPT Linux swap GUID
+  '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': true,
 }
 
-export const LVM_PARTITION_TYPE = 0x8e
+// MBR LVM type
+export const LVM_PARTITION_TYPE_MBR = 0x8e
+// GPT LVM type
+export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
 
 const parsePartxLine = createParser({
   keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
-  valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
+  valueTransform: (value, key) => {
+    if (key === 'start' || key === 'size') {
+      return +value
+    }
+    // For GPT partitions, type is a UUID string
+    if (key === 'type' && !value.startsWith('0x')) {
+      return value.toLowerCase()
+    }
+    // For MBR partitions, type is a hex number as string (e.g., "0x8e")
+    if (key === 'type' && value.startsWith('0x')) {
+      return parseInt(value, 16)
+    }
+    return value
+  },
 })
 
 // returns an empty array in case of a non-partitioned disk

@xen-orchestra/backups/_otherConfig.mjs

@@ -43,7 +43,7 @@ async function getDeltaChainLength(xapi, type, ref) {
 }
 
 /**
- * set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
+ * set the delta chain length ( number of delta since last base backup) to a VM and its associated VDIs
  *
  * @param {Xapi} xapi
  * @param {String} vmRef
@@ -58,7 +58,7 @@ export async function setVmDeltaChainLength(xapi, vmRef, length) {
 
 /**
  * Compute the delta chain length of a VM and its associated VDIs
- * if there is a discrependcy, use, the highest value
+ * if there is a discrepancy, use, the highest value
  * @param {Xapi} xapi
  * @param {String} vmRef
  * @returns {Promise}
@@ -70,7 +70,8 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 
 /**
  *
- * Reset the other_config field of a VM and its VDIs
+ * Reset the other_config field related to backups of a VM and its VDIs
+ *
  *
  * @param {Xapi} xapi
  * @param {String} vmRef
@@ -90,39 +91,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
   })
 }
 
-/**
- *
- * used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export async function populateVdisOtherConfig(xapi, vmRef) {
-  const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
-  const {
-    [DATETIME]: datetime,
-    [DELTA_CHAIN_LENGTH]: chainLength,
-    [EXPORTED_SUCCESSFULLY]: successfully,
-    [JOB_ID]: jobId,
-    [REPLICATED_TO_SR_UUID]: replicatedTo,
-    [SCHEDULE_ID]: scheduleId,
-    [VM_UUID]: vmUuid,
-  } = otherConfig
-
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
-    xapi.setFieldEntries(type, ref, 'other_config', {
-      [DATETIME]: datetime,
-      [DELTA_CHAIN_LENGTH]: chainLength,
-      [EXPORTED_SUCCESSFULLY]: successfully,
-      [JOB_ID]: jobId,
-      [REPLICATED_TO_SR_UUID]: replicatedTo,
-      [SCHEDULE_ID]: scheduleId,
-      [VM_UUID]: vmUuid,
-    })
-  )
-}
-
 /**
  *
  * set the other_config key related to a backup of a VM and its associated VDIs
@@ -152,7 +120,7 @@ export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, schedule
 }
 /**
  *
- * mark the export of he VM and its VDIs as successfull
+ * mark the export of he VM and its VDIs as successful
  *
  * @param {Xapi} xapi
  * @param {String} vmRef

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -4,11 +4,12 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
-import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
+import { Throttle } from '@vates/generator-toolbox'
+import createStreamThrottle from './_createStreamThrottle.mjs'
 
 const noop = Function.prototype
 
@@ -41,6 +42,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
+    const throttleGenerator = new Throttle(settings.maxExportRate)
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
@@ -89,7 +91,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             schedule,
             settings: vmSettings,
             sourceRemoteAdapter,
-            throttleStream,
+            throttleGenerator, // for incrementals
+            throttleStream, // for full
             vmUuid,
           }
           let vmBackup

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -4,12 +4,13 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
-import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
 import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
+import { Throttle } from '@vates/generator-toolbox'
+import createStreamThrottle from './_createStreamThrottle.mjs'
 
 const noop = Function.prototype
 
@@ -55,6 +56,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
+    const throttleGenerator = new Throttle(settings.maxExportRate)
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
@@ -147,6 +149,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
                       schedule,
                       settings: vmSettings,
                       srs,
+                      throttleGenerator,
                       throttleStream,
                       vm,
                     }

@xen-orchestra/backups/_runners/_vmRunners/FullRemote.mjs

@@ -2,10 +2,11 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
 
 export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
-  _getRemoteWriter() {
-    return FullRemoteWriter
+  _getRemoteWriters() {
+    return [FullRemoteWriter, AggregatedFullRemoteWriter]
   }
 
   _filterTransferList(transferList) {
@@ -16,7 +17,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
     for (const metadata of transferList) {
-      const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+      const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
       const sizeContainer = watchStreamSize(stream)
 
       // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
@@ -24,8 +25,9 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
         writer =>
           writer.run({
             stream: forkStreamUnpipe(stream),
-            // stream will be forked and transformed, it's not safe to attach additionnal properties to it
+            // stream will be forked and transformed, it's not safe to attach additional properties to it
             streamLength: stream.length,
+            maxStreamLength: stream.maxStreamLength, // on encrypted source
             timestamp: metadata.timestamp,
             vm: metadata.vm,
             vmSnapshot: metadata.vmSnapshot,
@@ -36,5 +38,6 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
       // for healthcheck
       this._tags = metadata.vm.tags
     }
+    this._hasTransferredData = transferList.length > 0
   }
 }

@xen-orchestra/backups/_runners/_vmRunners/FullXapi.mjs

@@ -5,23 +5,45 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
 import { AbstractXapi } from './_AbstractXapi.mjs'
+import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
+import { AggregatedFullXapiWriter } from '../_writers/AggregatedFullXapiWriter.mjs'
 
 const { debug } = createLogger('xo:backups:FullXapiVmBackup')
 
 export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
-    return [FullRemoteWriter, FullXapiWriter]
+    return [FullRemoteWriter, FullXapiWriter, AggregatedFullRemoteWriter, AggregatedFullXapiWriter]
   }
 
-  _mustDoSnapshot() {
+  async _mustDoSnapshot() {
     const vm = this._vm
-
     const settings = this._settings
-    return (
-      settings.unconditionalSnapshot ||
-      (!settings.offlineBackup && vm.power_state === 'Running') ||
-      settings.snapshotRetention !== 0
-    )
+
+    // General setting forcing snapshot at all times
+    if (settings.unconditionalSnapshot) {
+      return true
+    }
+
+    // If offline backup option is not checked
+    // and vm is running, we make a snapshot
+    if (!settings.offlineBackup && vm.power_state === 'Running') {
+      return true
+    }
+
+    // General setting used for snapshot retention
+    // see schedule in backupJob edition
+    if (settings.snapshotRetention !== 0) {
+      return true
+    }
+
+    const vdis = await this._xapi.getRecords('VDI', await vm.$getDisks())
+    const hasNOBAKtag = vdis.some(idx => idx.name_label.includes('[NOBAK]'))
+
+    // if there is a disk with NOBAK tag
+    // we need to make a snapshot
+    if (hasNOBAKtag) {
+      return true
+    }
   }
   _selectBaseVm() {}
 
@@ -43,10 +65,17 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
     for (const vdiRef of vdis) {
       const vdi = await this._xapi.getRecord('VDI', vdiRef)
 
-      // the size a of fully allocated vdi will be virtual_size  exaclty, it's a gross over evaluation
-      // of the real stream size in general, since a disk is never completly full
+      // the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
+      // of the real stream size in general, since a disk is rarely completely full
       // vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
-      maxStreamLength += vdi.virtual_size
+
+      // xva files are tar archive with data cut in 1MB blocks
+      // each block is signed with a hash (in xxhash format)
+      // each of these blocks + hash add one tar header entry in the file
+      const XVA_BLOCK_SIZE = 1024 * 1024
+      const nbBlocks = Math.ceil(vdi.virtual_size / XVA_BLOCK_SIZE)
+      const headersSize = nbBlocks * (512 /* file header */ + 512 /* xxhash header */ + 512) /* xhxhash size */
+      maxStreamLength += XVA_BLOCK_SIZE * nbBlocks + headersSize
     }
 
     const sizeContainer = watchStreamSize(stream)

@xen-orchestra/backups/_runners/_vmRunners/IncrementalRemote.mjs

@@ -3,8 +3,6 @@ import { createLogger } from '@xen-orchestra/log'
 import { asyncEach } from '@vates/async-each'
 import assert from 'node:assert'
 import * as UUID from 'uuid'
-import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
-import mapValues from 'lodash/mapValues.js'
 
 import { AbstractRemote } from './_AbstractRemote.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
@@ -12,14 +10,16 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
 import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
+import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
+import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
 
 const { warn } = createLogger('xo:backups:Incrementalremote')
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
-  _getRemoteWriter() {
-    return IncrementalRemoteWriter
+  _getRemoteWriters() {
+    return [IncrementalRemoteWriter, AggregatedIncrementalRemoteWriter]
   }
 
-  // we'll transfer the full list if at least one backup should be transfered
+  // we'll transfer the full list if at least one backup should be transferred
   // to ensure we don't cut the delta chain
   _filterTransferList(transferList) {
     if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
@@ -37,7 +37,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       return
     }
     await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
-      const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
+      const isDifferencing = metadata.isVhdDifferencing[id]
       if (isDifferencing) {
         const vmDir = getVmBackupDir(metadata.vm.uuid)
         const path = `${vmDir}/${metadata.vhds[id]}`
@@ -71,9 +71,13 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       // recompute if disks are differencing or not
       const isVhdDifferencing = {}
 
-      await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-        isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
-      })
+      for (const key in incrementalExport.disks) {
+        let disk = incrementalExport.disks[key]
+        isVhdDifferencing[key] = disk.isDifferencing()
+        disk = new ThrottledDisk(disk, this._throttleGenerator)
+        incrementalExport.disks[key] = new SynchronizedDisk(disk)
+      }
+
       const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
       if (metadata.isBase === hasDifferencingDisk) {
         warn(`Metadata isBase and real disk value are different`, {
@@ -87,11 +91,10 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       await this._selectBaseVm(metadata)
       await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
 
-      incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
       await this._callWriters(
         writer =>
           writer.transfer({
-            deltaExport: forkDeltaExport(incrementalExport),
+            deltaExport: forkDeltaExport(incrementalExport, writer.constructor.name),
             isVhdDifferencing,
             timestamp: metadata.timestamp,
             vm: metadata.vm,
@@ -114,6 +117,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       // for healthcheck
       this._tags = metadata.vm.tags
     }
+    this._hasTransferredData = transferList.length > 0
   }
 }
 

@xen-orchestra/backups/_runners/_vmRunners/IncrementalXapi.mjs

@@ -1,18 +1,12 @@
-import { asyncEach } from '@vates/async-each'
 import { createLogger } from '@xen-orchestra/log'
-import { pipeline } from 'node:stream'
-import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import keyBy from 'lodash/keyBy.js'
-import mapValues from 'lodash/mapValues.js'
-import vhdStreamValidator from 'vhd-lib/vhdStreamValidator.js'
 
 import { AbstractXapi } from './_AbstractXapi.mjs'
-import { exportIncrementalVm } from '../../_incrementalVm.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
+import { exportIncrementalVm } from '../../_incrementalVm.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
 import { Task } from '../../Task.mjs'
-import { watchStreamSize } from '../../_watchStreamSize.mjs'
 import {
   DATETIME,
   DELTA_CHAIN_LENGTH,
@@ -20,16 +14,23 @@ import {
   setVmDeltaChainLength,
   markExportSuccessfull,
 } from '../../_otherConfig.mjs'
+import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
+import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
+import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
-const noop = Function.prototype
 
 export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
-    return [IncrementalRemoteWriter, IncrementalXapiWriter]
+    return [
+      IncrementalRemoteWriter,
+      IncrementalXapiWriter,
+      AggregatedIncrementalRemoteWriter,
+      AggregatedIncrementalXapiWriter,
+    ]
   }
 
-  _mustDoSnapshot() {
+  async _mustDoSnapshot() {
     return true
   }
 
@@ -45,42 +46,37 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       nbdConcurrency: this._settings.nbdConcurrency,
       preferNbd: this._settings.preferNbd,
     })
-    // since NBD is network based, if one disk use nbd , all the disk use them
-    // except the suspended VDI
-    if (Object.values(deltaExport.streams).some(({ _nbd }) => _nbd)) {
+
+    const isVhdDifferencing = {}
+    let useNbd = false
+    for (const key in deltaExport.disks) {
+      let disk = deltaExport.disks[key]
+      isVhdDifferencing[key] = disk.isDifferencing()
+      if (!isFull && !isVhdDifferencing[key] && key !== exportedVm.$suspend_VDI?.$ref) {
+        Task.warning('Backup fell back to a full')
+      }
+      useNbd = useNbd || disk.useNbd()
+      disk = new ThrottledDisk(disk, this._throttleGenerator)
+      deltaExport.disks[key] = new SynchronizedDisk(disk)
+    }
+    if (useNbd) {
       Task.info('Transfer data using NBD')
     }
 
-    const isVhdDifferencing = {}
-    // since isVhdDifferencingDisk is reading and unshifting data in stream
-    // it should be done BEFORE any other stream transform
-    await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
-      isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
-    })
-    const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
-
-    if (this._settings.validateVhdStreams) {
-      deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
-    }
-    deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
+    // @todo : reimplement throttle
 
     const timestamp = Date.now()
-
     await this._callWriters(
       writer =>
         writer.transfer({
-          deltaExport: forkDeltaExport(deltaExport),
+          deltaExport: forkDeltaExport(deltaExport, writer.constructor.name),
           isVhdDifferencing,
-          sizeContainers,
           timestamp,
           vm,
           vmSnapshot: exportedVm,
         }),
       'writer.transfer()'
     )
-
-    // we want to control the uuid of the vhd in the chain
-    // and ensure they are correctly chained
     await this._callWriters(
       writer =>
         writer.updateUuidAndChain({
@@ -102,15 +98,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       await markExportSuccessfull(this._xapi, exportedVm.$ref)
     }
 
-    const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
-    const end = Date.now()
-    const duration = end - timestamp
-    debug('transfer complete', {
-      duration,
-      speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
-      size,
-    })
-
     await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
   }
 

@xen-orchestra/backups/_runners/_vmRunners/_Abstract.mjs

@@ -74,9 +74,12 @@ export const Abstract = class AbstractVmBackupRunner {
 
     // check if current VM has tags
     const tags = this._tags
-    const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
 
-    if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
+    // accept both 'xo:no-health-check' and 'xo:no-health-check=reason'
+    const vmAlwaysIgnored = tags.some(t => t.startsWith('xo:no-health-check'))
+    const intersect = !vmAlwaysIgnored && settings.healthCheckVmsWithTags.some(t => tags.includes(t))
+
+    if (vmAlwaysIgnored || (settings.healthCheckVmsWithTags.length !== 0 && !intersect)) {
       // create a task to have an info in the logs and reports
       return Task.run(
         {