update readme and componentstory to use new menu components

Introduced by 320a61323

.eslintrc.js

@@ -39,6 +39,12 @@ module.exports = {
             version: '>=16',
           },
         ],
+        'n/no-unsupported-features/es-builtins': [
+          'error',
+          {
+            version: '>=16',
+          },
+        ],
         'n/no-unsupported-features/es-syntax': [
           'error',
           {
@@ -99,11 +105,15 @@ module.exports = {
             ],
           },
         ],
+        'no-useless-constructor': 'off',
+        '@typescript-eslint/no-useless-constructor': 'error',
         'no-void': ['error', { allowAsStatement: true }],
         'n/no-missing-import': 'off', // using 'import' plugin instead, to support TS aliases
         'no-redeclare': 'off', // automatically checked by the TypeScript compiler
         'no-dupe-class-members': 'off', // automatically checked by the TypeScript compiler
         'no-undef': 'off', // automatically checked by the TypeScript compiler
+        'no-use-before-define': 'off', // replaced with the rule below
+        '@typescript-eslint/no-use-before-define': ['error', { functions: false }],
         '@typescript-eslint/no-explicit-any': 'off',
         'vue/multi-word-component-names': 'off',
         // Vue 3 - Strongly Recommended
@@ -138,6 +148,7 @@ module.exports = {
         'vue/no-v-html': 'error',
         'vue/this-in-template': 'error',
         // Vue 3 - Uncategorized
+        'vue/block-lang': ['error', { script: { lang: 'ts' }, style: { lang: 'postcss' } }],
         'vue/block-order': ['error', { order: ['template', 'script', 'style'] }],
         'vue/block-tag-newline': ['error', { singleline: 'consistent', multiline: 'always', maxEmptyLines: 0 }],
         'vue/component-api-style': ['error', ['script-setup']],
@@ -216,7 +227,7 @@ module.exports = {
         '@intlify/vue-i18n/no-v-html': 'error',
         '@intlify/vue-i18n/valid-message-syntax': 'error',
         '@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
-        '@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa'] }],
+        '@intlify/vue-i18n/no-missing-keys-in-other-locales': ['error', { ignoreLocales: ['de', 'fa', 'cs'] }],
       },
     },
     // Specific rules for XO dev-related files
@@ -252,6 +263,8 @@ module.exports = {
     },
   ],
 
+  ignorePatterns: ['@vates/fatfs/'],
+
   parserOptions: {
     ecmaVersion: 13,
     sourceType: 'script',

.github/workflows/ci.yml

@@ -5,34 +5,46 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # https://github.com/actions/checkout
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - name: Install packages
         run: |
           sudo apt-get update
           sudo apt-get install -y curl qemu-utils python3-vmdkstream git libxml2-utils libfuse2 nbdkit
+
+      # https://turbo.build/repo/docs/guides/ci-vendors/github-actions#caching-with-github-actionscache
       - name: Cache Turbo
-        # https://github.com/actions/cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
-          path: '**/node_modules/.cache/turbo'
-          key: ${{ runner.os }}-turbo-cache
+          path: '.turbo'
+          key: ${{ runner.os }}-turbo-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-turbo-
+
+      # https://github.com/actions/setup-node
       - name: Setup Node environment
-        # https://github.com/actions/setup-node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
           cache: 'yarn'
+
       - name: Install project dependencies
         run: yarn
+
       - name: Ensure packages are sorted in CHANGELOG.unreleased.md
         run: ./scripts/gen-deps-list.js --check-order > /dev/null
+
       - name: Ensure yarn.lock is up-to-date
         run: git diff --exit-code yarn.lock
+
       - name: Build the project
         run: yarn build
+
       - name: Unit tests
         run: yarn test-unit
+
       - name: Lint tests
         run: yarn test-lint
+
       - name: Integration tests
         run: sudo yarn test-integration

@vates/async-each/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it, beforeEach } = require('test')
+const { describe, it, beforeEach } = require('node:test')
 const assert = require('assert').strict
 const { spy } = require('sinon')
 

@vates/async-each/package.json

@@ -30,10 +30,9 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
-    "test": "^3.2.1"
+    "sinon": "^18.0.0"
   }
 }

@vates/coalesce-calls/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('assert')
 
 const { coalesceCalls } = require('./')

@vates/coalesce-calls/package.json

@@ -31,9 +31,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.2.1"
+    "test": "node --test"
   }
 }

@vates/compose/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('node:assert').strict
 
 const { compose } = require('./')

@vates/compose/package.json

@@ -20,9 +20,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.2.1"
+    "test": "node --test"
   }
 }

@vates/decorate-with/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert')
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 
 const { decorateClass, decorateWith, decorateMethodsWith, perInstance } = require('./')
 

@vates/decorate-with/package.json

@@ -26,9 +26,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.2.1"
+    "test": "node --test"
   }
 }

@vates/diff/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 const diff = require('./index.js')
 

@vates/diff/package.json

@@ -28,9 +28,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
+    "test": "node --test"
   }
 }

@vates/disposable/debounceResource.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const { useFakeTimers, spy, assert } = require('sinon')
 
 const { createDebounceResource } = require('./debounceResource')

@vates/disposable/deduped.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const { spy, assert } = require('sinon')
 
 const { deduped } = require('./deduped')

@vates/disposable/package.json

@@ -14,22 +14,21 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "engines": {
     "node": ">=8.10"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   },
   "dependencies": {
     "@vates/multi-key-map": "^0.2.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/log": "^0.7.1",
     "ensure-array": "^1.0.0"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
-    "test": "^3.2.1"
+    "sinon": "^18.0.0"
   }
 }

@vates/fatfs/.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+.vscode/
+node_modules/

@vates/fatfs/README.md

@@ -0,0 +1,127 @@
+# @vates/fatfs
+
+This is a fork of the original library [natevw/fatfs](https://github.com/natevw/fatfs).
+
+## Why?
+
+This fork was created to add a missing feature in the original library: the ability to create labels (createLabel).
+The original library did not support this functionality, which was required to make Cloudbase-Init work on windows VMs.
+See: https://github.com/natevw/fatfs/issues/30, https://github.com/natevw/fatfs/pull/31
+
+A standalone FAT16/FAT32 implementation that takes in a block-access interface and exposes something quite similar to `require('fs')` (i.e. the node.js built-in [Filesystem API](http://nodejs.org/api/fs.html)).
+
+## Installation
+
+`npm install fatfs`
+
+## Example
+
+```js
+var fatfs = require('fatfs'),
+  fs = fatfs.createFileSystem(exampleDriver) // see below
+fs.stat('autoexec.bat', function (e, stats) {
+  if (e) console.error(e)
+  else console.log(stats)
+})
+// TODO: open a file and write to it or something…
+```
+
+## API
+
+- `fs = fatfs.createFileSystem(vol, [opts], [cb])` — Simply pass in a block driver (see below) mapped to a FAT partition somewhere, and get back the API documented [here](http://nodejs.org/api/fs.html). An options dictionary can be provided, details are in the next section. You may also optionally provide a callback `cb(err)` which will be automatically registered for the on `'ready'` or `'error'` event.
+- `'ready'` event — fired on `fs` when initial volume information has been determined and the API is ready to use. It is safe to call other `fs` methods before this fires **only if** you are sure the first sector will be readable and represents a valid FAT volume.
+- `'error'` event — fired if initialization fails for whatever reason.
+
+### Filesystem options
+
+The `opts` dictionary you pass to `fatfs.createFileSystem` can contain any of the following options:
+
+- `ro` — Enables readonly mode if `true`. It defaults to `false`, but if your volume driver does not provide a `writeSectors` method it will be overriden to `true`.
+- `noatime` — The FAT filesystem can track the last access time (just a date, actually) but this means every read operation would also incur some write overhead. Defaults to `true`, meaning by default access times will **not** be updated on reads. Set this to `false` to track access times.
+- `modmode` — chooses how `fs.chmod` (and the mode field from `fs.stat`–family calls) should map FAT attributes to POSIX permissions. Set to the number `0111` to map the readonly flag to the user's write bit being unset, and the archive/system/hidden flags to the user/group/other executable bits respectively. Set to the number `07000` to map the readonly flag to _all_ write bits being unset, and the archive/system/hidden flags to the sticky/setgid/setuid bits respectively. Set to `null` for readonly mapping. Defaults to `0111`.
+- `umask` — any bits _set_ in this octal number will be _unset_ in the 'mode' field from `fs.stat`–family calls. It does not affect anything else. Defaults to `process.umask()`, or `0022` if that is unavailable.
+- `uid` — This value will be returned as the 'uid' stat field. It does not affect anything else. Defaults to `process.getuid()`, or `0` if that is unavailable.
+- `gid` — This value will be returned as the 'gid' stat field. It does not affect anything else. Defaults to `process.getgid()`, or `0` if that is unavailable.
+
+(Note that these are similar to the options you could use with a POSIX `mount` operation.)
+
+And that's it! The [rest of the API](http://nodejs.org/api/fs.html) (`fs.readdir`, `fs.open`, `fs.createReadStream`, `fs.appendFile`, etc.) is as documented by the node.js project.
+
+Well, sort of…
+
+## Caveats
+
+### Temporary
+
+- **BETA** **BETA** **BETA**. Seriously, this is a _brand new_, _from scratch_, _completely unproven_ filesystem implementation. It does not have full automated test coverage, and it has not been manually tested very much either. Please please please **make sure you have a backup** of any important drive/image/card you unleash this upon.
+- A few methods are not quite implemented, either: `fs.rename`, `fs.unlink` and `fs.rmdir`, as well as `fs.watchFile`/`fs.unwatchFile` and `fs.watch`. These are Coming Soon™.
+- There are several internal housekeeping items (redundant FAT tables, extra FAT32 information, etc.) that are not done. These do not seem to affect interop, but you may see warnings when repairing a filesystem written by this module.
+- Oh, and not to scare you, but if an IO error happens while writing, the library usually just bails — bubbling an error up to your callback as if it were a hot potato. Although some attempt has been made to do separate writes in the safest order (e.g. allocating an additional file cluster, then appending data into it, and then finally updating the file's size), but this behavior has not been thoroughly audited for all operations. There's certainly no attempt to retry/cleanup/rollback if a multi-step change runs into trouble partway through.
+
+### As-planned
+
+Some of the differences between `fatfs` and the node.js `fs` module are "by design" for architectural simplicity and/or due to underlying FAT limitations.
+
+- There are no `fs.*Sync` methods. (The volume driver is async, not to mention that supporting a separate \*Sync codepath would be an enormous duplication of effort of dubious value.)
+- This module does [almost] no read/write caching. This should be done in your volume driver, but see notes below.
+- You'll need multiple `createFileSystem` instances for multiple volumes; paths are relative to each, and don't share a namespace.
+- The FAT filesystem has no concept of symlinks, and hardlinks are not really an intentional feature. You will get an ENOSYS-like error when trying to create either type of link.
+
+## "Volume driver" API
+
+To use 'fatfs', you must provide a driver object with the following properties/methods:
+
+- `driver.sectorSize` — number of bytes per sector on this device
+- `driver.numSectors` — count of sectors available on this media
+- `driver.readSectors(i, dest, cb)` — Fill `dest` with data starting at the `i`th sector and notify `cb(e)` when finished. You may assume `dest.length` is a multiple of `driver.sectorSize`.
+- `driver.writeSectors(i, data, cb)` — (optional) Write `data` starting at the `i`th sector and notify `cb(e)` when finished. You may assume `data.length` is a multiple of `driver.sectorSize`.
+
+If you do not provide a `writeSectors` method, then `fatfs` will work in readonly mode. Pretty simple, eh? And the 'fatfs' module makes a good effort to check the parameters passed to your driver methods!
+
+**TBD:** to facilitate proper cache handling, this module might add an optional `driver.flush(cb)` method at some point in the future.
+
+Here's an example taken from code used to run this module's own tests:
+
+```js
+// NOTE: this assumes image at `path` has no partition table.
+//       If it did, you'd need to translate positions, natch…
+var fs = require('fs')
+
+exports.createDriverSync = function (path, opts) {
+  opts || (opts = {})
+
+  var secSize = 512,
+    ro = opts.readOnly || false,
+    fd = fs.openSync(path, ro ? 'r' : 'r+'),
+    s = fs.fstatSync(fd)
+
+  return {
+    sectorSize: secSize,
+    numSectors: s.size / secSize,
+    readSectors: function (i, dest, cb) {
+      if (dest.length % secSize) throw Error('Unexpected buffer length!')
+      fs.read(fd, dest, 0, dest.length, i * secSize, function (e, n, d) {
+        cb(e, d)
+      })
+    },
+    writeSectors: ro
+      ? null
+      : function (i, data, cb) {
+          if (data.length % secSize) throw Error('Unexpected buffer length!')
+          fs.write(fd, data, 0, data.length, i * secSize, function (e) {
+            cb(e)
+          })
+        },
+  }
+}
+```
+
+## License
+
+© 2014 Nathan Vander Wilt.
+Funding for this work was provided by Technical Machine, Inc.
+
+Reuse under your choice of:
+
+- [BSD-2-Clause](http://opensource.org/licenses/BSD-2-Clause)
+- [Apache 2.0](http://www.apache.org/licenses/LICENSE-2.0.html)

@vates/fatfs/cache.js

@@ -0,0 +1,60 @@
+var _ = require("./helpers.js");
+
+exports.wrapDriver = function (volume, opts) {
+    opts = _.extend({
+        maxSectors: 2048
+    }, opts);
+    
+    var cache = {},
+        advice = 'NORMAL',
+        secSize = volume.sectorSize;
+    
+    function _freezeBuffer(b) {
+        var f = _.allocBuffer(b.length);
+        b.copy(f);
+        return f;
+    }
+    
+    function addToCache(i, data) {
+        if (advice === 'SEQUENTIAL' || advice === 'NOREUSE') return;
+        data = _freezeBuffer(data);
+        cache[i] = data;
+        //if (data.length > secSize) addToCache(i+1, data.slice(secSize));
+        while (data.length > secSize) {
+            data = data.slice(secSize);
+            cache[++i] = data;
+        }
+        // simple highest-sectors-lose eviction policy for now
+        Object.keys(cache).sort().slice(opts.maxSectors).forEach(function (x) {
+            delete cache[x];
+        });
+        _.log(_.log.DBG, "Cache now contains:", Object.keys(cache).join(','));
+    }
+    
+    return {
+        sectorSize: volume.sectorSize,
+        numSectors: volume.numSectors,
+        advice: function (val) {
+            if (!arguments.length) return advice;
+            else advice = val;
+            if (advice === 'SEQUENTIAL' || advice === 'NOREUSE') cache = {};
+            return this;
+        },
+        readSectors: function (i, dest, cb) {
+            // TODO: handle having partial parts of dest!
+            if (i in cache && dest.length === secSize) {
+                cache[i].copy(dest);
+                setImmediate(cb);
+            } else volume.readSectors(i, dest, function (e) {
+                if (e) cb(e);
+                else addToCache(i, dest), cb();
+            });
+        },
+        writeSectors: (!volume.writeSectors) ? null : function (i, data, cb) {
+            volume.writeSectors(i, data, function (e) {
+                if (e) cb(e);
+                else addToCache(i, data), cb();
+            });
+        }
+    };
+};

@vates/fatfs/chains.js

@@ -0,0 +1,287 @@
+var S = require("./structs.js"),
+    _ = require("./helpers.js");
+
+function _baseChain(vol) {
+    var chain = {};
+    
+    chain.sectorSize = vol._sectorSize;
+    
+    function posFromOffset(off) {
+        var secSize = chain.sectorSize,
+            offset = off % secSize,
+            sector = (off - offset) / secSize;
+        return {sector:sector, offset:offset};
+    }
+    
+    var sectorCache = vol._makeCache();
+    Object.defineProperty(chain, 'cacheAdvice', {
+        enumerable: true,
+        get: function () { return sectorCache.advice(); },
+        set: function (v) { sectorCache.advice(v); }
+    });
+    chain._vol_readSectors = vol._readSectors.bind(vol, sectorCache);
+    chain._vol_writeSectors = vol._writeSectors.bind(vol, sectorCache);
+    
+    // cb(error, bytesRead, buffer)
+    chain.readFromPosition = function (targetPos, buffer, cb) {
+        if (typeof targetPos === 'number') targetPos = posFromOffset(targetPos);
+        if (typeof buffer === 'number') buffer = _.allocBuffer(buffer);
+        /* NOTE: to keep our contract with the volume driver, we need to read on _full_ sector boundaries!
+                 So we divide the read into [up to] three parts: {preface, main, trailer}
+                 This is kind of unfortunate, but in practice should often still be reasonably efficient. */
+        if (targetPos.offset) chain.readSectors(targetPos.sector, _.allocBuffer(chain.sectorSize), function (e,d) {
+            if (e || !d) cb(e, 0, buffer);
+            else {  // copy preface into `buffer`
+                var dBeg = targetPos.offset,
+                    dEnd = dBeg + buffer.length;
+                d.copy(buffer, 0, dBeg, dEnd);
+                if (dEnd > d.length) readMain();
+                else cb(null, buffer.length, buffer);
+            }
+        }); else readMain();
+        function readMain() {
+            var prefaceLen = targetPos.offset && (chain.sectorSize - targetPos.offset),
+                trailerLen = (buffer.length - prefaceLen) % chain.sectorSize,
+                mainSector = (prefaceLen) ? targetPos.sector + 1 : targetPos.sector,
+                mainBuffer = (trailerLen) ? buffer.slice(prefaceLen, -trailerLen) : buffer.slice(prefaceLen);
+            if (mainBuffer.length) chain.readSectors(mainSector, mainBuffer, function (e,d) {
+                if (e || !d) cb(e, prefaceLen, buffer);
+                else if (!trailerLen) cb(null, buffer.length, buffer);
+                else readTrailer();
+            }); else readTrailer();
+            function readTrailer() {
+                var trailerSector = mainSector + (mainBuffer.length / chain.sectorSize);
+                chain.readSectors(trailerSector, _.allocBuffer(chain.sectorSize), function (e,d) {
+                    if (e || !d) cb(e, buffer.length-trailerLen, buffer);
+                    else {
+                        d.copy(buffer, buffer.length-trailerLen, 0, trailerLen);
+                        cb(null, buffer.length, buffer);
+                    }
+                });
+            }
+        }
+    };
+    
+    // cb(error)
+    chain.writeToPosition = function (targetPos, data, cb) {
+        _.log(_.log.DBG, "WRITING", data.length, "bytes at", targetPos, "in", this.toJSON(), data);
+        if (typeof targetPos === 'number') targetPos = posFromOffset(targetPos);
+        
+        var prefaceBuffer = (targetPos.offset) ? data.slice(0, chain.sectorSize-targetPos.offset) : null;
+        if (prefaceBuffer) _modifySector(targetPos.sector, targetPos.offset, prefaceBuffer, function (e) {
+            if (e) cb(e);
+            else if (prefaceBuffer.length < data.length) writeMain();
+            else cb();
+        }); else writeMain();
+        function writeMain() {
+            var prefaceLen = (prefaceBuffer) ? prefaceBuffer.length : 0,
+                trailerLen = (data.length - prefaceLen) % chain.sectorSize,
+                mainSector = (prefaceLen) ? targetPos.sector + 1 : targetPos.sector,
+                mainBuffer = (trailerLen) ? data.slice(prefaceLen, -trailerLen) : data.slice(prefaceLen);
+            if (mainBuffer.length) chain.writeSectors(mainSector, mainBuffer, function (e) {
+                if (e) cb(e);
+                else if (!trailerLen) cb();
+                else writeTrailer();
+            }); else writeTrailer();
+            function writeTrailer() {
+                var trailerSector = mainSector + (mainBuffer.length / chain.sectorSize),
+                    trailerBuffer = data.slice(data.length-trailerLen);     // WORKAROUND: https://github.com/tessel/runtime/issues/721
+                _modifySector(trailerSector, 0, trailerBuffer, cb);
+            }
+        }
+        function _modifySector(sec, off, data, cb) {
+            chain.readSectors(sec, _.allocBuffer(chain.sectorSize), function (e, orig) {
+                if (e) return cb(e);
+                orig || (orig = _.allocBuffer(chain.sectorSize, 0));
+                data.copy(orig, off);
+                chain.writeSectors(sec, orig, cb);
+            });
+        }
+    };
+    
+    return chain;
+};
+
+
+
+exports.clusterChain = function (vol, firstCluster, _parent) {
+    var chain = _baseChain(vol),
+        cache = [firstCluster];
+    
+    chain.firstCluster = firstCluster;
+    
+    function _cacheIsComplete() {
+        return cache[cache.length-1] === 'eof';
+    }
+    
+    function extendCacheToInclude(i, cb) {          // NOTE: may `cb()` before returning!
+        if (i < cache.length) cb(null, cache[i]);
+        else if (_cacheIsComplete()) cb(null, 'eof');
+        else vol.fetchFromFAT(cache[cache.length-1], function (e,d) {
+            if (e) cb(e);
+            else if (typeof d === 'string' && d !== 'eof') cb(S.err.IO());
+            else {
+                cache.push(d);
+                extendCacheToInclude(i, cb);
+            }
+        });
+    }
+    
+    function expandChainToLength(clusterCount, cb) {
+        if (!_cacheIsComplete()) throw Error("Must be called only when cache is complete!");
+        else cache.pop();            // remove 'eof' entry until finished
+        
+        function addCluster(clustersNeeded, lastCluster) {
+            if (!clustersNeeded) cache.push('eof'), cb();
+            else vol.allocateInFAT(lastCluster, function (e, newCluster) {
+                if (e) cb(e);
+                else vol.storeToFAT(lastCluster, newCluster, function (e) {
+                    if (e) return cb(e);
+                    
+                    cache.push(newCluster);
+                    addCluster(clustersNeeded-1, newCluster);
+                });
+            });
+        }
+        addCluster(clusterCount - cache.length, cache[cache.length - 1]);
+    }
+    
+    function shrinkChainToLength(clusterCount, cb) {
+        if (!_cacheIsComplete()) throw Error("Must be called only when cache is complete!");
+        else cache.pop();            // remove 'eof' entry until finished
+        
+        function removeClusters(count, cb) {
+            if (!count) cache.push('eof'), cb();
+            else vol.storeToFAT(cache.pop(), 'free', function (e) {
+                if (e) cb(e);
+                else removeClusters(count - 1, cb);
+            });
+        }
+        // NOTE: for now, we don't remove the firstCluster ourselves; we should though!
+        if (clusterCount) removeClusters(cache.length - clusterCount, cb);
+        else removeClusters(cache.length - 1, cb);
+    }
+    
+    // [{firstSector,numSectors},{firstSector,numSectors},…]
+    function determineSectorGroups(sectorIdx, numSectors, alloc, cb) {
+        var sectorOffset = sectorIdx % vol._sectorsPerCluster,
+            clusterIdx = (sectorIdx - sectorOffset) / vol._sectorsPerCluster,
+            numClusters = Math.ceil((numSectors + sectorOffset) / vol._sectorsPerCluster),
+            chainLength = clusterIdx + numClusters;
+        extendCacheToInclude(chainLength-1, function (e,c) {
+            if (e) cb(e);
+            else if (c === 'eof' && alloc) expandChainToLength(chainLength, function (e) {
+                if (e) cb(e);
+                else _determineSectorGroups();
+            });
+            else _determineSectorGroups();
+        });
+        function _determineSectorGroups() {
+            // …now we have a complete cache
+            var groups = [],
+                _group = null;
+            for (var i = clusterIdx; i < chainLength; ++i) {
+                var c = (i < cache.length) ? cache[i] : 'eof';
+                if (c === 'eof') break;
+                else if (_group && c !== _group._nextCluster) {
+                    groups.push(_group);
+                    _group = null;
+                }
+                if (!_group) _group = {
+                   _nextCluster: c+1,
+                   firstSector: vol._firstSectorOfCluster(c) + sectorOffset,
+                   numSectors: vol._sectorsPerCluster - sectorOffset
+                }; else {
+                    _group._nextCluster += 1;
+                    _group.numSectors += vol._sectorsPerCluster;
+                }
+                sectorOffset = 0;       // only first group is offset
+            }
+            if (_group) groups.push(_group);
+            cb(null, groups, i === chainLength);
+        }
+    }
+    
+    chain.readSectors = function (i, dest, cb) {
+        var groupOffset = 0, groupsPending;
+        determineSectorGroups(i, dest.length / chain.sectorSize, false, function (e, groups, complete) {
+            if (e) cb(e);
+            else if (!complete) groupsPending = -1, _pastEOF(cb);
+            else if ((groupsPending = groups.length)) groups.forEach(function (group) {
+                var groupLength = group.numSectors * chain.sectorSize,
+                    groupBuffer = dest.slice(groupOffset, groupOffset += groupLength);
+                chain._vol_readSectors(group.firstSector, groupBuffer, function (e,d) {
+                    if (e && groupsPending !== -1) groupsPending = -1, cb(e);
+                    else if (--groupsPending === 0) cb(null, dest);
+                });
+            });
+            else cb(null, dest);     // 0-length destination case
+        });
+    };
+    
+    // TODO: does this handle NOSPC condition?
+    chain.writeSectors = function (i, data, cb) {
+        var groupOffset = 0, groupsPending;
+        determineSectorGroups(i, data.length / chain.sectorSize, true, function (e, groups) {
+            if (e) cb(e);
+            else if ((groupsPending = groups.length)) groups.forEach(function (group) {
+                var groupLength = group.numSectors * chain.sectorSize,
+                    groupBuffer = data.slice(groupOffset, groupOffset += groupLength);
+                chain._vol_writeSectors(group.firstSector, groupBuffer, function (e) {
+                    if (e && groupsPending !== -1) groupsPending = -1, cb(e);
+                    else if (--groupsPending === 0) cb();
+                });
+            });
+            else cb();     // 0-length data case
+        });
+    };
+    
+    chain.truncate = function (numSectors, cb) {
+        extendCacheToInclude(Infinity, function (e,c) {
+            if (e) return cb(e);
+            
+            var currentLength = cache.length-1,
+                clustersNeeded = Math.ceil(numSectors / vol._sectorsPerCluster);
+            if (clustersNeeded < currentLength) shrinkChainToLength(clustersNeeded, cb);
+            else if (clustersNeeded > currentLength) expandChainToLength(clustersNeeded, cb);
+            else cb();
+        });
+    };
+    
+    
+    chain.toJSON = function () {
+        return {firstCluster:firstCluster};
+    };
+    
+    return chain;
+};
+
+exports.sectorChain = function (vol, firstSector, numSectors) {
+    var chain = _baseChain(vol);
+    
+    chain.firstSector = firstSector;
+    chain.numSectors = numSectors;
+    
+    chain.readSectors = function (i, dest, cb) {
+        if (i < numSectors) chain._vol_readSectors(firstSector+i, dest, cb);
+        else _pastEOF(cb);
+    };
+    
+    chain.writeSectors = function (i, data, cb) {
+        if (i < numSectors) chain._vol_writeSectors(firstSector+i, data, cb);
+        else _.delayedCall(cb, S.err.NOSPC());
+    };
+    
+    chain.truncate = function (i, cb) {
+        _.delayedCall(cb, S.err.INVAL());
+    };
+    
+    chain.toJSON = function () {
+        return {firstSector:firstSector, numSectors:numSectors};
+    };
+    
+    return chain;
+};
+
+// NOTE: used with mixed feelings, broken out to mark uses
+function _pastEOF(cb) { _.delayedCall(cb, null, null); }

@vates/fatfs/dir.js

@@ -0,0 +1,372 @@
+var S = require("./structs.js"),
+    _ = require("./helpers.js");
+
+var dir = exports;
+
+dir.iterator = function (dirChain, opts) {
+    opts || (opts = {});
+    
+    var cache = {buffer:null, n: null};
+    function getSectorBuffer(n, cb) {
+        if (cache.n === n) cb(null, cache.buffer);
+        else cache.n = cache.buffer = null, dirChain.readSectors(n, _.allocBuffer(dirChain.sectorSize), function (e,d) {
+            if (e) cb(e);
+            else if (!d) return cb(null, null);
+            else {
+                cache.n = n;
+                cache.buffer = d;
+                getSectorBuffer(n, cb);
+            }
+        });
+    }
+    
+    var secIdx = 0,
+        off = {bytes:0},
+        long = null;
+    function getNextEntry(cb) {
+        if (off.bytes >= dirChain.sectorSize) {
+            secIdx += 1;
+            off.bytes -= dirChain.sectorSize;
+        }
+        var entryPos = {chain:dirChain, sector:secIdx, offset:off.bytes};
+        getSectorBuffer(secIdx, function (e, buf) {
+            if (e) return cb(S.err.IO());
+            else if (!buf) return cb(null, null, entryPos);
+            
+            var entryIdx = off.bytes,
+                signalByte = buf[entryIdx];
+            if (signalByte === S.entryDoneFlag) return cb(null, null, entryPos);
+            else if (signalByte === S.entryFreeFlag) {
+                off.bytes += S.dirEntry.size;
+                long = null;
+                if (opts.includeFree) return cb(null, {_free:true,_pos:entryPos}, entryPos);
+                else return getNextEntry(cb);       // usually just skip these
+            }
+            
+            var attrByte = buf[entryIdx+S.dirEntry.fields.Attr.offset],
+                entryType = (attrByte === S.longDirFlag) ? S.longDirEntry : S.dirEntry_simple;
+            var entry = entryType.valueFromBytes(buf, off);
+            entry._pos = entryPos;
+            _.log(_.log.DBG, "entry:", entry, secIdx, entryIdx);
+            if (entryType === S.longDirEntry) {
+                var firstEntry;
+                if (entry.Ord & S.lastLongFlag) {
+                    firstEntry = true;
+                    entry.Ord &= ~S.lastLongFlag;
+                    long = {
+                        name: -1,
+                        sum: entry.Chksum,
+                        _rem: entry.Ord-1,
+                        _arr: []
+                    }
+                }
+                if (firstEntry || long && entry.Chksum === long.sum && entry.Ord === long._rem--) {
+                    var S_lde_f = S.longDirEntry.fields,
+                        namepart = entry.Name1;
+                    if (entry.Name1.length === S_lde_f.Name1.size/2) {
+                        namepart += entry.Name2;
+                        if (entry.Name2.length === S_lde_f.Name2.size/2) {
+                            namepart += entry.Name3;
+                        }
+                    }
+                    long._arr.push(namepart);
+                    if (!long._rem) {
+                        long.name = long._arr.reverse().join('');
+                        delete long._arr;
+                        delete long._rem;
+                    }
+                } else long = null;
+            } else if ((attrByte & 0x08) === 0) {        // NOTE: checks `!entry.Attr.volume_id`
+                var bestName = null;
+                if (long && long.name) {
+                    var pos = entryIdx + S.dirEntry.fields['Name'].offset,
+                        sum = _.checksumName(buf, pos);
+                    if (sum === long.sum) bestName = long.name;
+                }
+                if (!bestName) {
+                    if (signalByte === S.entryIsE5Flag) entry.Name.filename = '\u00E5'+entry.Name.filename.slice(1);
+                    
+                    var nam = entry.Name.filename.replace(/ +$/, ''),
+                        ext = entry.Name.extension.replace(/ +$/, '');
+                    // TODO: lowercase bits http://en.wikipedia.org/wiki/8.3_filename#Compatibility
+                    //       via NTRes, bits 0x08 and 0x10 http://www.fdos.org/kernel/fatplus.txt.1
+                    bestName = (ext) ? nam+'.'+ext : nam;
+                }
+                entry._name = bestName;
+                
+                // OPTIMIZATION: avoid processing any fields for non-matching entries
+                // TODO: we could make this automatic via getters, but…?
+                var _entryBuffer = buf.slice(off.bytes-S.dirEntry.size, off.bytes);
+                entry._full = function () {
+                    var _entry = S.dirEntry.valueFromBytes(_entryBuffer);
+                    _.extend(entry, _entry);
+                    entry._size = entry.FileSize;
+                    entry._firstCluster = (entry.FstClusHI << 16) + entry.FstClusLO;
+                    return entry;
+                };
+                
+                long = null;
+                return cb(null, entry, entryPos);
+            } else long = null;
+            getNextEntry(cb);
+        });
+    }
+    
+    function iter(cb) {
+        getNextEntry(cb);
+        return iter;            // TODO: previous value can't be re-used, so why make caller re-assign?
+    }
+    return iter;
+};
+
+function _updateEntry(vol, entry, newStats) {
+    if ('size' in newStats) entry._size = entry.FileSize = newStats.size;
+    
+    if ('_touch' in newStats) newStats.archive = newStats.atime = newStats.mtime = true;
+    if ('archive' in newStats) entry.Attr.archive = true;           // TODO: also via newStats.mode?
+    
+    var _now;
+    function applyDate(d, prefix, timeToo, tenthToo) {
+        if (d === true) d = _now || (_now = new Date());
+        entry[prefix+'Date'] = {year:d.getFullYear()-1980, month:d.getMonth()+1, day:d.getDate()};
+        if (timeToo) {
+            entry[prefix+'Time'] = {hours:d.getHours(), minutes:d.getMinutes(), seconds_2:d.getSeconds()>>>1};
+            if (tenthToo) {
+                var msec = (d.getSeconds() % 2)*1000 + d.getMilliseconds();
+                entry[prefix+'TimeTenth'] = Math.floor(msec / 100);
+            }
+        }
+    }
+    if ('ctime' in newStats) applyDate(newStats.ctime, 'Crt', true, true);
+    if ('mtime' in newStats) applyDate(newStats.mtime, 'Wrt', true);
+    if ('atime' in newStats) applyDate(newStats.atime, 'LstAcc');
+    
+    if ('mode' in newStats) {
+        entry.Attr.directory = (newStats.mode & S._I.FDIR) ? true : false;
+        entry.Attr.volume_id = (newStats.mode & S._I.FREG) ? false : true;
+        if (vol.opts.modmode === 0111) {
+            entry.Attr.archive = (newStats.mode & S._I.XUSR) ? true : false;
+            entry.Attr.system  = (newStats.mode & S._I.XGRP) ? true : false;
+            entry.Attr.hidden  = (newStats.mode & S._I.XOTH) ? true : false;
+            entry.Attr.readonly = (newStats.mode & S._I.WUSR) ? false : true;
+        } else if (vol.opts.modmode === 07000) {
+            entry.Attr.archive = (newStats.mode & S._I.SVTX) ? true : false;
+            entry.Attr.system  = (newStats.mode & S._I.SGID) ? true : false;
+            entry.Attr.hidden  = (newStats.mode & S._I.SUID) ? true : false;
+            entry.Attr.readonly = (
+                newStats.mode & S._I.WUSR ||
+                newStats.mode & S._I.WGRP ||
+                newStats.mode & S._I.WOTH
+            ) ? false : true;
+        }
+    }
+    
+    if ('firstCluster' in newStats) {
+        entry.FstClusLO = newStats.firstCluster & 0xFFFF;
+        entry.FstClusHI = newStats.firstCluster >>> 16;
+        entry._firstCluster = newStats.firstCluster;
+    }
+    return entry;
+}
+
+dir.makeStat = function (vol, entry) {
+    var stats = {};     // TODO: return an actual `instanceof fs.Stat` somehow?
+    
+    stats.isFile = function () {
+        return (!entry.Attr.volume_id && !entry.Attr.directory);
+    };
+    stats.isDirectory = function () {
+        return entry.Attr.directory;
+    };
+    stats.isBlockDevice = function () { return false; }
+    stats.isCharacterDevice = function () { return false; }
+    stats.isSymbolicLink = function () { return false; }
+    stats.isFIFO = function () { return false; }
+    stats.isSocket = function () { return false; }
+    stats.size = entry.FileSize;
+    stats.blksize = vol._sectorsPerCluster*vol._sectorSize;
+    stats.blocks = Math.ceil(stats.size / stats.blksize) || 1;
+    stats.nlink = 1;
+    
+    stats.mode = S._I.RUSR | S._I.RGRP | S._I.ROTH;
+    if (!entry.Attr.readonly) stats.mode |= S._I.WUSR | S._I.WGRP | S._I.WOTH;
+    if (entry.Attr.directory) stats.mode |= S._I.FDIR;
+    else if (!entry.Attr.volume_id) stats.mode |= S._I.FREG;
+    // NOTE: discussion at https://github.com/natevw/fatfs/issues/7
+    if (vol.opts.modmode === 0111) {
+        // expose using executable bits, like Samba
+        if (entry.Attr.archive) stats.mode |= S._I.XUSR;
+        if (entry.Attr.system)  stats.mode |= S._I.XGRP;
+        if (entry.Attr.hidden)  stats.mode |= S._I.XOTH;
+    } else if (vol.opts.modmode === 07000) {
+        // expose using setXid/sticky bits, like MKS
+        if (entry.Attr.archive) stats.mode |= S._I.SVTX;
+        if (entry.Attr.system)  stats.mode |= S._I.SGID;
+        if (entry.Attr.hidden)  stats.mode |= S._I.SUID;
+    }
+    
+    stats.mode &= ~vol.opts.umask;
+    stats.uid =  vol.opts.uid;
+    stats.gid = vol.opts.gid;
+    
+    function extractDate(prefix) {
+        var date = entry[prefix+'Date'],
+            time = entry[prefix+'Time'] || {hours:0, minutes:0, seconds_2:0},
+            secs = time.seconds_2 * 2,
+            sect = entry[prefix+'TimeTenth'] || 0;
+        if (sect > 100) {
+            secs += 1;
+            sect -= 100;
+        }
+        return new Date(date.year+1980, date.month-1, date.day, time.hours, time.minutes, secs, sect*100);
+    }
+    stats.atime = extractDate('LstAcc');
+    stats.mtime = extractDate('Wrt');
+    stats.ctime = extractDate('Crt');
+    
+    entry = {           // keep immutable copy (with only the fields we need)
+        Attr: _.extend({},entry.Attr),
+    };
+    
+    return stats;
+};
+
+dir.init = function (vol, dirInfo, cb) {
+    var dirChain = dirInfo.chain,
+        isRootDir = ('numSectors' in dirChain),    // HACK: all others would be a clusterChain
+        initialCluster = _.allocBuffer(dirChain.sectorSize*vol._sectorsPerCluster),
+        entriesOffset = {bytes:0};
+    initialCluster.fill(0);
+    function writeEntry(name, clusterNum) {
+        while (name.length < 8) name += " ";
+        S.dirEntry.bytesFromValue(_updateEntry(vol, {
+            Name: {filename:name, extension:"   "},
+            Attr: {directory:true}
+        }, {firstCluster:clusterNum, _touch:true,ctime:true}), initialCluster, entriesOffset);
+    }
+    if (!isRootDir) {
+        writeEntry(".", dirChain.firstCluster);
+        writeEntry("..", dirInfo.parent.chain.firstCluster);
+    };
+    dirChain.writeToPosition(0, initialCluster, cb);
+};
+
+dir.addFile = function (vol, dirChain, entryInfo, opts, cb) {
+    if (typeof opts === 'function') {
+        cb = opts;
+        opts = {};
+    }
+    var name = entryInfo.name,
+        entries = [], mainEntry;
+    entries.push(mainEntry = {
+        Name: _.shortname(name),
+        Attr: {directory:opts.dir||false},
+        _name: name
+    });
+    if (1 || mainEntry.Name._lossy) {         // HACK: always write long names until `._lossy` is more useful!
+        var workaroundTessel427 = ('\uFFFF'.length !== 1);
+        if (workaroundTessel427) throw Error("Your JS runtime does not have proper Unicode string support. (If Tessel, is your firmware up-to-date?)");
+        
+        // name entries should be 0x0000-terminated and 0xFFFF-filled
+        var S_lde_f = S.longDirEntry.fields,
+            ENTRY_CHUNK_LEN = (S_lde_f.Name1.size + S_lde_f.Name2.size + S_lde_f.Name3.size)/2,
+            partialLen = name.length % ENTRY_CHUNK_LEN,
+            paddingNeeded = partialLen && (ENTRY_CHUNK_LEN - partialLen);
+        if (paddingNeeded--) name += '\u0000';
+        while (paddingNeeded-- > 0) name += '\uFFFF';
+        // now fill in as many entries as it takes
+        var off = 0,
+            ord = 1;
+        while (off < name.length) entries.push({
+            Ord: ord++,
+            Name1: name.slice(off, off+=S_lde_f.Name1.size/2),
+            Attr_raw: S.longDirFlag,
+            Chksum: null,
+            Name2: name.slice(off, off+=S_lde_f.Name2.size/2),
+            Name3: name.slice(off, off+=S_lde_f.Name3.size/2)
+        });
+        entries[entries.length - 1].Ord |= S.lastLongFlag;
+    }
+    
+    if (entryInfo.tail) {
+        var name = mainEntry.Name.filename,
+            suffix = '~'+entryInfo.tail,
+            endIdx = name.indexOf(' '),
+            sufIdx = (~endIdx) ? Math.min(endIdx, name.length-suffix.length) : name.length-suffix.length;
+        if (sufIdx < 0) return cb(S.err.NAMETOOLONG());         // TODO: would EXIST be more correct?
+        mainEntry.Name.filename = name.slice(0,sufIdx)+suffix+name.slice(sufIdx+suffix.length);
+        _.log(_.log.DBG, "Shortname amended to:", mainEntry.Name);
+    }
+    
+    vol.allocateInFAT(dirChain.firstCluster || 2, function (e,fileCluster) {
+        if (e) return cb(e);
+        
+        var nameBuf = S.dirEntry.fields['Name'].bytesFromValue(mainEntry.Name),
+            nameSum = _.checksumName(nameBuf);
+        // TODO: finalize initial properties… (via `opts.mode` instead?)
+        _updateEntry(vol, mainEntry, {firstCluster:fileCluster, size:0, ctime:true,_touch:true});
+        mainEntry._pos = _.adjustedPos(vol, entryInfo.target, S.dirEntry.size*(entries.length-1));
+        entries.slice(1).forEach(function (entry) {
+            entry.Chksum = nameSum;
+        });
+        entries.reverse();
+        if (entryInfo.lastEntry) entries.push({});
+        
+        var entriesData = _.allocBuffer(S.dirEntry.size*entries.length),
+            dataOffset = {bytes:0};
+        entries.forEach(function (entry) {
+            var entryType = ('Ord' in entry) ? S.longDirEntry : S.dirEntry;
+            entryType.bytesFromValue(entry, entriesData, dataOffset);
+        });
+        
+        _.log(_.log.DBG, "Writing", entriesData.length, "byte directory entry", mainEntry, "into", dirChain.toJSON(), "at", entryInfo.target);
+        dirChain.writeToPosition(entryInfo.target, entriesData, function (e) {
+            // TODO: if we get error, what/should we clean up?
+            if (e) cb(e);
+            else cb(null, mainEntry, vol.chainForCluster(fileCluster, dirChain));
+        });
+    });
+};
+
+dir.findInDirectory = function (vol, dirChain, name, opts, cb) {
+    var matchName = name.toUpperCase(),
+        tailName = (opts.prepareForCreate) ? _.shortname(name) : null,
+        maxTail = 0;
+    
+    function processNext(next) {
+        next = next(function (e, d, entryPos) {
+            if (e) cb(e);
+            else if (!d) cb(S.err.NOENT(), {tail:maxTail, target:entryPos, lastEntry:true});
+            else if (d._free) processNext(next);         // TODO: look for long enough reusable run
+            else if (d._name.toUpperCase() === matchName) return cb(null, d._full());
+            else if (!opts.prepareForCreate) processNext(next);
+            else {
+                var dNum = 1,
+                    dName = d.Name.filename,
+                    dTail = dName.match(/(.*)~(\d+)/);
+                if (dTail) {
+                    dNum = +dTail[2];
+                    dName = dTail[1];
+                }
+                if (tailName.extension === d.Name.extension &&
+                    tailName.filename.indexOf(dName) === 0)
+                {
+                    maxTail = Math.max(dNum+1, maxTail);
+                }
+                processNext(next);
+            }
+        });
+    }
+    processNext(dir.iterator(dirChain, {includeFree:(0 && opts.prepareForCreate)}));
+};
+
+dir.updateEntry = function (vol, entry, newStats, cb) {
+    if (!entry._pos || !entry._pos.chain) throw Error("Entry source unknown!");
+    
+    var entryPos = entry._pos,
+        newEntry = _updateEntry(vol, entry, newStats),
+        data = S.dirEntry.bytesFromValue(newEntry);
+    _.log(_.log.DBG, "UPDATING ENTRY", newStats, newEntry, entryPos, data);
+    // TODO: if write fails, then entry becomes corrupt!
+    entryPos.chain.writeToPosition(entryPos, data, cb);
+};

@vates/fatfs/helpers.js

@@ -0,0 +1,206 @@
+var S = require("./structs.js"),
+    _xok = require('xok');
+
+// ponyfills for older node.js
+exports.allocBuffer = Buffer.alloc || function (len, val) {
+    var b = Buffer(len);
+    if (arguments.length > 1) b.fill(val);
+    return b;
+};
+exports.bufferFrom = Buffer.from || function (arg0, arg1) {
+    return (arguments.length > 1) ? Buffer(arg0, arg1) : Buffer(arg0);
+};
+
+// flag for WORKAROUND: https://github.com/tessel/beta/issues/380
+exports.workaroundTessel380 = !Buffer.from && function () {
+    var b = Buffer([0]),
+        s = b.slice(0);
+    return ((s[0] = 0xFF) !== b[0]);
+}();
+
+
+// WORKAROUND: https://github.com/tessel/beta/issues/433
+var oldslice;
+if (!Buffer.alloc && Buffer(5).slice(10).length < 0) oldslice = Buffer.prototype.slice, Buffer.prototype.slice = function (s, e) {
+    if (s > this.length) s = this.length;
+    // ~WORKAROUND: https://github.com/tessel/beta/issues/434
+    return (arguments.length > 1) ? oldslice.call(this, s, e) : oldslice.call(this, s);
+}
+
+exports.absoluteSteps = function (path) {
+    var steps = [];
+    path.split('/').forEach(function (str) {
+        // NOTE: these should actually be fine, just wasteful…
+        if (str === '..') steps.pop();
+        else if (str && str !== '.') steps.push(str);
+    });
+    return steps.map(exports.longname);
+};
+
+exports.absolutePath = function (path) {
+    return '/'+exports.absoluteSteps(path).join('/');
+};
+
+exports.parseFlags = function (flags) {
+    // read, write, append, create, truncate, exclusive
+    var info, _dir;           // NOTE: there might be more clever ways to "parse", but…
+    if (flags[0] === '\\') {
+        // internal flag used internally to `fs.open` directories without `S.err.ISDIR()`
+        flags = flags.slice(1);
+        _dir = true;
+    }
+    switch (flags) {
+        case 'r':   info = {read:true, write:false, create:false}; break;
+        case 'r+':  info = {read:true, write:true, create:false}; break;
+        case 'rs':  info = {read:true, write:false, create:false, sync:true}; break;
+        case 'rs+': info = {read:true, write:true, create:false, sync:true}; break;
+        case 'w':   info = {read:false, write:true, create:true, truncate:true}; break;
+        case 'wx':  info = {read:false, write:true, create:true, exclusive:true}; break;
+        case 'w+':  info = {read:true, write:true, create:true, truncate:true}; break;
+        case 'wx+': info = {read:true, write:true, create:true, exclusive:true}; break;
+        case 'a':   info = {read:false, write:true, create:true, append:true}; break;
+        case 'ax':  info = {read:false, write:true, create:true, append:true, exclusive:true}; break;
+        case 'a+':  info = {read:true, write:true, create:true, append:true}; break;
+        case 'ax+': info = {read:true, write:true, create:true, append:true, exclusive:true}; break;
+        default: throw Error("Uknown mode: "+flags);       // TODO: throw as `S.err.INVAL`
+    }
+    if (info.sync) throw Error("Mode not implemented.");    // TODO: what would this require of us?
+    if (_dir) info._openDir = true;
+    return info;
+};
+
+
+// TODO: these are great candidates for special test coverage!
+var _snInvalid = /[^A-Z0-9$%'-_@~`!(){}^#&.]/g;         // NOTE: '.' is not valid but we split it away
+exports.shortname = function (name) {
+    var lossy = false;
+    // TODO: support preservation of case for otherwise non-lossy name!
+    name = name.toUpperCase().replace(/ /g, '').replace(/^\.+/, '');
+    name = name.replace(_snInvalid, function () {
+        lossy = true;
+        return '_';
+    });
+    
+    var parts = name.split('.'),
+        basis3 = parts.pop(),
+        basis8 = parts.join('');
+    if (!parts.length) {
+        basis8 = basis3;
+        basis3 = '   ';
+    }
+    if (basis8.length > 8) {
+        basis8 = basis8.slice(0,8);
+        // NOTE: technically, spec's "lossy conversion" flag is NOT set by excess length.
+        //       But since lossy conversion and truncated names both need a numeric tail…
+        lossy = true;
+    } else while (basis8.length < 8) basis8 += ' ';
+    if (basis3.length > 3) {
+        basis3 = basis3.slice(0,3);
+        lossy = true;
+    } else while (basis3.length < 3) basis3 += ' ';
+    
+    return {filename:basis8, extension:basis3, _lossy:lossy};
+    return {basis:[basis8,basis3], lossy:lossy};
+};
+//shortname("autoexec.bat") => {basis:['AUTOEXEC','BAT'],lossy:false}
+//shortname("autoexecutable.batch") => {basis:['AUTOEXEC','BAT'],lossy:true}
+// TODO: OS X stores `shortname("._.Trashes")` as ['~1', 'TRA'] — should we?
+
+var _lnInvalid = /[^a-zA-Z0-9$%'-_@~`!(){}^#&.+,;=[\] ]/g;
+exports.longname = function (name) {
+    name = name.trim().replace(/\.+$/, '').replace(_lnInvalid, function (c) {
+        if (c.length > 1) throw Error("Internal problem: unexpected match length!");
+        if (c.charCodeAt(0) > 127) return c;
+        else throw Error("Invalid character "+JSON.stringify(c)+" in name.");
+        lossy = true;
+        return '_';
+    });
+    if (name.length > 255) throw Error("Name is too long.");
+    return name;
+};
+
+function nameChkSum(sum, c) {
+    return ((sum & 1) ? 0x80 : 0) + (sum >>> 1) + c & 0xFF;
+}
+
+// WORKAROUND: https://github.com/tessel/beta/issues/335
+function reduceBuffer(buf, start, end, fn, res) {
+    // NOTE: does not handle missing `res` like Array.prototype.reduce would
+    for (var i = start; i < end; ++i) {
+        res = fn(res, buf[i]);
+    }
+    return res;
+}
+
+exports.checksumName = function (buf,off) {
+    off || (off = 0);
+    var len = S.dirEntry.fields['Name'].size;
+    return reduceBuffer(buf, off, off+len, nameChkSum, 0);
+};
+
+
+/* comparing C rounding trick from FAT spec with Math.ceil
+function tryBoth(d) {
+    var a = ((D.RootEntCnt * 32) + (D.BytsPerSec - 1)) / D.BytsPerSec >>> 0,
+        b = Math.ceil((D.RootEntCnt * 32) / D.BytsPerSec);
+    if (b !== a) console.log("try", b, a, (b === a) ? '' : '*');
+    return (b === a);
+}
+// BytsPerSec — "may take on only the following values: 512, 1024, 2048 or 4096"
+[512, 1024, 2048, 4096].forEach(function (bps) {
+    // RootEntCnt — "a count that when multiplied by 32 results in an even multiple of BPB_BytsPerSec"
+    for (var evenMultiplier = 0; evenMultiplier < 1024*1024*16; evenMultiplier += 2) {
+        var rec = (bps * evenMultiplier) / 32;
+        tryBoth({RootEntCnt:rec, BytsPerSec:bps});
+    }
+});
+*/
+
+exports.fmtHex = function (n, ff) {
+    return (1+ff+n).toString(16).slice(1);
+};
+
+exports.delayedCall = function (fn) {
+    if (!fn) throw Error("No function provided!");      // debug aid
+    var ctx = this,
+        args = Array.prototype.slice.call(arguments, 1);
+    setImmediate(function () {
+        fn.apply(ctx, args);
+    });
+};
+
+exports.adjustedPos = function (vol, pos, bytes) {
+    var _pos = {
+        chain: pos.chain,
+        sector: pos.sector,
+        offset: pos.offset + bytes
+    }, secSize = vol._sectorSize;
+    while (_pos.offset >= secSize) {
+        _pos.sector += 1;
+        _pos.offset -= secSize;
+    }
+    return _pos;
+};
+
+exports.extend = _xok;
+
+var _prevDbg = Date.now(),
+    _thresh = 50;
+
+function log(level) {
+    if (level < log.level) return;
+    
+    var now = Date.now(),
+        diff = now - _prevDbg;
+    arguments[0] = ((diff < _thresh) ? " " : '') + diff.toFixed(0)  + "ms";
+    console.log.apply(console, arguments);
+    _prevDbg = now;
+}
+log.DBG = -4;
+log.INFO = -3;
+log.WARN = -2;
+log.ERR = -1;
+
+log.level = log.WARN;
+
+exports.log = log;

@vates/fatfs/img_volume.js

@@ -0,0 +1,25 @@
+var fs = require('fs');
+
+exports.createDriverSync = function (path, opts) {
+    opts || (opts = {});
+    
+    var secSize = 512,
+        ro = opts.readOnly || false,
+        fd = fs.openSync(path, (ro) ? 'r' : 'r+'),
+        s = fs.fstatSync(fd);
+    
+    return {
+        sectorSize: secSize,
+        numSectors: s.size / secSize,
+        readSectors: function (i, dest, cb) {
+            fs.read(fd, dest, 0, dest.length, i*secSize, function (e,n,d) {
+                cb(e,d);
+            });
+        },
+        writeSectors: (ro) ? null : function (i, data, cb) {
+            fs.write(fd, data, 0, data.length, i*secSize, function (e) {
+                cb(e);
+            });
+        }
+    };
+};

@vates/fatfs/index.js

@@ -0,0 +1,771 @@
+var events = require('events'),
+  streams = require('stream'),
+  fifolock = require('fifolock'),
+  S = require('./structs.js'),
+  _ = require('./helpers.js')
+
+//_.log.level = _.log.DBG;
+
+exports.createFileSystem = function (volume, opts, cb) {
+  if (typeof opts === 'function') {
+    cb = opts
+    opts = null
+  }
+  opts = _.extend(
+    {
+      // c.f. https://www.kernel.org/doc/Documentation/filesystems/vfat.txt
+      ro: false,
+      noatime: true,
+      modmode: 0111, // or `07000`
+      umask: 'umask' in process ? process.umask() : 0022,
+      uid: 'getuid' in process ? process.getuid() : 0,
+      gid: 'getgid' in process ? process.getgid() : 0,
+    },
+    opts
+  )
+  if (!volume.writeSectors) opts.ro = true
+  if (opts.ro) opts.noatime = true // natch
+
+  var fs = new events.EventEmitter(),
+    vol = null,
+    dir = require('./dir.js'),
+    c = require('./chains.js'),
+    q = fifolock()
+
+  var GROUP =
+    _.log.level < _.log.INFO
+      ? q.TRANSACTION_WRAPPER.bind({
+          postAcquire: function (proceed) {
+            _.log(_.log.DBG, '=== Starting GROUP ===')
+            proceed()
+          },
+          preRelease: function (finish) {
+            _.log(_.log.DBG, '=== Finishing GROUP ===')
+            finish()
+          },
+        })
+      : q.TRANSACTION_WRAPPER
+
+  q.acquire(function (unlock) {
+    // because of this, callers can start before 'ready'
+    var d = _.allocBuffer(volume.sectorSize)
+    volume.readSectors(0, d, function (e) {
+      if (e) fs.emit('error', e)
+      else {
+        try {
+          init(d)
+        } catch (e) {
+          fs.emit('error', e)
+          unlock()
+          return
+        }
+        fs.emit('ready')
+      }
+      unlock()
+    })
+  })
+
+  if (cb) fs.on('error', cb).on('ready', cb.bind(null, null))
+
+  function init(bootSector) {
+    vol = require('./vol.js').init(volume, opts, bootSector)
+    fs._dirIterator = dir.iterator.bind(dir)
+
+    var entryInfoByPath = {},
+      baseEntry = {
+        _refs: 0,
+        _record: function () {
+          entryInfoByPath[this.path] = this
+          if (this.parent) this.parent.retain()
+          return this
+        },
+        retain: function () {
+          if (!this._refs) this._record()
+          this._refs += 1
+          return this
+        },
+        release: function () {
+          this._refs -= 1
+          if (!this._refs) this._rescind()
+        },
+        _rescind: function () {
+          if (this.parent) this.parent.release()
+          delete entryInfoByPath[this.path]
+        },
+      }
+    fs._createSharedEntry = function (path, entry, chain, parent) {
+      return _.extend(Object.create(baseEntry), {
+        _refs: 0, // WORKAROUND: https://github.com/tessel/beta/issues/455
+        path: path,
+        entry: entry,
+        chain: chain,
+        parent: parent,
+      }).retain()
+    }
+    fs._createSharedEntry('/', { Attr: { directory: true } }, vol.rootDirectoryChain)
+    fs._sharedEntryForSteps = function (steps, opts, cb) {
+      // NOTE: may `cb` before returning!
+      var path = steps.join('/') || '/',
+        name = steps.pop(), // n.b.
+        info = entryInfoByPath[path]
+      if (info) cb(null, info.retain())
+      else
+        fs._sharedEntryForSteps(steps, {}, function (e, parentInfo) {
+          // n.b. `steps` don't include `name`
+          if (e) cb(e)
+          else if (!parentInfo.entry.Attr.directory) cb(S.err.NOTDIR())
+          else
+            dir.findInDirectory(vol, parentInfo.chain, name, opts, function (e, entry) {
+              if (e && !opts.prepareForCreate) cb(e)
+              else if (e) cb(e, { missingChild: _.extend(entry, { name: name }), parent: parentInfo })
+              else cb(null, fs._createSharedEntry(path, entry, vol.chainForCluster(entry._firstCluster), parentInfo))
+            })
+        })
+    }
+
+    fs._updateEntry = dir.updateEntry.bind(dir, vol)
+    fs._makeStat = dir.makeStat.bind(dir, vol)
+    fs._addFile = dir.addFile.bind(dir, vol)
+    fs._initDir = dir.init.bind(dir, vol)
+  }
+
+  /**** ---- CORE API ---- ****/
+
+  // NOTE: we really don't share namespace, but avoid first three anyway…
+  var fileDescriptors = [null, null, null]
+
+  fs.open = function (path, flags, mode, cb, _n_) {
+    if (typeof mode === 'function') {
+      _n_ = cb
+      cb = mode
+      mode = 0666
+    }
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = { flags: null, entry: null, chain: null, pos: 0 },
+          f = _.parseFlags(flags)
+        if (vol.opts.ro && (f.write || f.create || f.truncate)) return _.delayedCall(cb, S.err.ROFS())
+        else _fd.flags = f
+
+        fs._sharedEntryForSteps(_.absoluteSteps(path), { prepareForCreate: f.create }, function (e, info) {
+          if (e && !(e.code === 'NOENT' && f.create && info)) cb(e)
+          else if (e)
+            fs._addFile(info.parent.chain, info.missingChild, { dir: f._openDir }, function (e, newEntry, newChain) {
+              if (e) cb(e)
+              else finish(fs._createSharedEntry(_.absolutePath(path), newEntry, newChain, info.parent))
+            })
+          else if (info && f.exclusive) cb(S.err.EXIST())
+          else if (info.entry.Attr.directory && !f._openDir) cb(S.err.ISDIR())
+          else if (f.write && info.entry.Attr.readonly) cb(S.err.ACCES())
+          else finish(info)
+          function finish(fileInfo) {
+            var fd = fileDescriptors.push(_fd) - 1
+            _fd.info = fileInfo
+            _fd.entry = fileInfo.entry
+            _fd.chain = fileInfo.chain
+            if (f.append) _fd.pos = _fd.entry._size
+            if (f._openDir) _fd.chain.cacheAdvice = 'WILLNEED'
+            if (f.truncate && _fd.entry._size)
+              fs.ftruncate(
+                fd,
+                0,
+                function (e) {
+                  cb(e, fd)
+                },
+                '_nested_'
+              )
+            else _.delayedCall(cb, null, fd) // (delay in case fs._sharedEntryForSteps all cached!)
+          }
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.fstat = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) _.delayedCall(cb, S.err.BADF())
+        else _.delayedCall(cb, null, fs._makeStat(_fd.entry))
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.futimes = function (fd, atime, mtime, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        // NOTE: ctime would get touched on POSIX; but we map that to create time!
+        else fs._updateEntry(_fd.entry, { atime: atime || true, mtime: mtime || true }, cb)
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.fchmod = function (fd, mode, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else {
+          mode &= S._I._chmoddable
+          if (_fd.entry.Attr.directory) mode |= S._I.FDIR
+          else if (!_fd.entry.Attr.volume_id) mode |= S._I.FREG
+          fs._updateEntry(_fd.entry, { mode: mode }, cb)
+        }
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.read = function (fd, buf, off, len, pos, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) return _.delayedCall(cb, S.err.BADF())
+
+        var _pos = pos === null ? _fd.pos : pos,
+          _len = Math.min(len, _fd.entry._size - _pos),
+          _buf = buf.slice(off, off + _len)
+        _fd.chain.readFromPosition(_pos, _buf, function (e, bytes, slice) {
+          if (_.workaroundTessel380) _buf.copy(buf, off) // WORKAROUND: https://github.com/tessel/beta/issues/380
+          _fd.pos = _pos + bytes
+          if (e || vol.opts.noatime) finish(e)
+          else fs._updateEntry(_fd.entry, { atime: true }, finish)
+          function finish(e) {
+            cb(e, bytes, buf)
+          }
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs._readdir = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.read) return _.delayedCall(cb, S.err.BADF())
+
+        var entryNames = [],
+          getNextEntry = fs._dirIterator(_fd.chain)
+        function processNext() {
+          getNextEntry(function (e, d) {
+            if (e) cb(e)
+            else if (!d && !entryNames.length)
+              cb(null, entryNames) // WORKAROUND: https://github.com/tessel/beta/issues/435
+            else if (!d)
+              cb(null, entryNames.sort()) // NOTE: sort not required, but… [simplifies tests for starters!]
+            else {
+              if (d._name !== '.' && d._name !== '..') entryNames.push(d._name)
+              processNext()
+            }
+          })
+        }
+        processNext()
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs._mkdir = function (fd, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else fs._initDir(_fd.info, cb)
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.write = function (fd, buf, off, len, pos, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) return _.delayedCall(cb, S.err.BADF())
+
+        var _pos = pos === null || _fd.flags.append ? _fd.pos : pos,
+          _buf = buf.slice(off, off + len)
+        if (_pos > _fd.entry._size) {
+          // TODO: handle huge jumps by zeroing clusters individually?
+          var padLen = _pos - _fd.entry._size,
+            padBuf = _.allocBuffer(padLen + _buf.length)
+          padBuf.fill(0x00, 0, padLen)
+          _buf.copy(padBuf, padLen)
+          _pos = _fd.entry._size
+          _buf = padBuf
+        }
+        _fd.chain.writeToPosition(_pos, _buf, function (e) {
+          _fd.pos = _pos + len
+          var newSize = Math.max(_fd.entry._size, _fd.pos),
+            newInfo = { size: newSize, _touch: true }
+          fs._updateEntry(_fd.entry, newInfo, function (ee) {
+            cb(e || ee, len, buf)
+          })
+        })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  fs.ftruncate = function (fd, len, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) return _.delayedCall(cb, S.err.BADF())
+
+        var newStats = { size: len, _touch: true }
+        // NOTE: we order operations for best state in case of only partial success
+        if (len === _fd.entry._size) _.delayedCall(cb)
+        else if (len < _fd.entry._size)
+          fs._updateEntry(_fd.entry, newStats, function (e) {
+            if (e) cb(e)
+            else _fd.chain.truncate(Math.ceil(len / _fd.chain.sectorSize), cb)
+          })
+        // TODO: handle huge file expansions without as much memory pressure
+        else
+          _fd.chain.writeToPosition(_fd.entry._size, _.allocBuffer(len - _fd.entry._size, 0x00), function (e) {
+            if (e) cb(e)
+            else fs._updateEntry(_fd.entry, newStats, cb)
+          })
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  // 'NORMAL', 'SEQUENTIAL', 'RANDOM', 'WILLNEED', 'DONTNEED', 'NOREUSE'
+  fs._fadviseSync = function (fd, off, len, advice) {
+    if (off !== 0 || len !== 0) throw Error('Cache advise can currently be given only for whole file!')
+    var _fd = fileDescriptors[fd]
+    if (!_fd) throw S.err.BADF()
+    else _fd.chain.cacheAdvice = advice
+  }
+
+  fs.fsync = function (fd, cb) {
+    // NOTE: we'll need to flush write cache here once we have one…
+    var _fd = fileDescriptors[fd]
+    if (!_fd) _.delayedCall(cb, S.err.BADF())
+    else _.delayedCall(cb)
+  }
+
+  fs.close = function (fd, cb) {
+    var _fd = fileDescriptors[fd]
+    if (!_fd) _.delayedCall(cb, S.err.BADF())
+    else setTimeout(_fd.info.release.bind(_fd.info), 500), _.delayedCall(cb, (fileDescriptors[fd] = null))
+  }
+
+  /* STREAM WRAPPERS */
+
+  var workaroundTessel436
+  try {
+    new require('stream').Readable({ encoding: 'utf8' })
+    new streams.Readable({ encoding: 'utf8' })
+  } catch (e) {
+    workaroundTessel436 = true
+  }
+
+  function _createStream(StreamType, path, opts) {
+    // [NOT REALLY A] WORKAROUND: https://github.com/tessel/beta/issues/436
+    if (workaroundTessel436 && 'encoding' in opts) {
+      console.warn('Tessel does not currently support encoding option for Readable streams, discarding!')
+      delete opts.encoding
+    }
+
+    var fd = opts.fd !== null ? opts.fd : '_opening_',
+      pos = opts.start,
+      stream = new StreamType(opts)
+
+    if (fd === '_opening_')
+      fs.open(path, opts.flags, opts.mode, function (e, _fd) {
+        if (e) {
+          fd = '_open_error_'
+          stream.emit('error', e)
+        } else {
+          fd = _fd
+          fs._fadviseSync(fd, 0, 0, 'SEQUENTIAL')
+          stream.emit('open', fd)
+        }
+      })
+
+    function autoClose(tombstone) {
+      // NOTE: assumes caller will clear `fd`
+      if (opts.autoClose)
+        fs.close(fd, function (e) {
+          if (e) stream.emit('error', e)
+          else stream.emit('close')
+        })
+      fd = tombstone
+    }
+
+    if (StreamType === streams.Readable) {
+      stream._read = function (n) {
+        var buf
+        // TODO: optimize to fetch at least a full sector regardless of `n`…
+        n = Math.min(n, opts.end - pos)
+        if (fd === '_opening_')
+          stream.once('open', function () {
+            stream._read(n)
+          })
+        else if (pos > opts.end) stream.push(null)
+        else if (n > 0)
+          (buf = _.allocBuffer(n)),
+            fs.read(fd, buf, 0, n, pos, function (e, n, d) {
+              if (e) {
+                autoClose('_read_error_')
+                stream.emit('error', e)
+              } else stream.push(n ? d.slice(0, n) : null)
+            }),
+            (pos += n)
+        else stream.push(null)
+      }
+
+      stream.once('end', function () {
+        autoClose('_ended_')
+      })
+    } else if (StreamType === streams.Writable) {
+      stream.bytesWritten = 0
+
+      stream._write = function (data, _enc, cb) {
+        if (fd === '_opening_')
+          stream.once('open', function () {
+            stream._write(data, null, cb)
+          })
+        else
+          fs.write(fd, data, 0, data.length, pos, function (e, n) {
+            if (e) {
+              autoClose('_write_error_')
+              cb(e)
+            } else {
+              stream.bytesWritten += n
+              cb()
+            }
+          }),
+            (pos += data.length)
+      }
+
+      stream.once('finish', function () {
+        autoClose('_finished_')
+      })
+    }
+
+    return stream
+  }
+
+  fs.createReadStream = function (path, opts) {
+    return _createStream(
+      streams.Readable,
+      path,
+      _.extend(
+        {
+          start: 0,
+          end: Infinity,
+          flags: 'r',
+          mode: 0666,
+          encoding: null,
+          fd: null, // ??? see https://github.com/joyent/node/issues/7708
+          autoClose: true,
+        },
+        opts
+      )
+    )
+  }
+
+  fs.createWriteStream = function (path, opts) {
+    return _createStream(
+      streams.Writable,
+      path,
+      _.extend(
+        {
+          start: 0,
+          flags: 'w',
+          mode: 0666,
+          //encoding: null,   // see https://github.com/joyent/node/issues/7710
+          fd: null, // ??? see https://github.com/joyent/node/issues/7708
+          autoClose: true,
+        },
+        opts,
+        { decodeStrings: true, objectMode: false }
+      )
+    )
+  }
+
+  /* PATH WRAPPERS (albeit the only public interface for some folder operations) */
+
+  function _fdOperation(path, opts, fn, cb) {
+    cb = GROUP(cb, function () {
+      opts.advice || (opts.advice = 'NORMAL')
+      fs.open(
+        path,
+        opts.flag,
+        function (e, fd) {
+          if (e) cb(e)
+          else
+            fs._fadviseSync(fd, 0, 0, opts.advice),
+              fn(fd, function () {
+                var ctx = this,
+                  args = arguments
+                fs.close(
+                  fd,
+                  function (closeErr) {
+                    cb.apply(ctx, args)
+                  },
+                  '_nested_'
+                )
+              })
+        },
+        '_nested_'
+      )
+    })
+  }
+
+  fs.stat = fs.lstat = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r' },
+      function (fd, cb) {
+        fs.fstat(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.exists = function (path, cb) {
+    fs.stat(path, function (err) {
+      cb(err ? false : true)
+    })
+  }
+
+  fs.readFile = function (path, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'r')
+    opts.advice || (opts.advice = 'NOREUSE')
+    _fdOperation(
+      path,
+      opts,
+      function (fd, cb) {
+        fs.fstat(
+          fd,
+          function (e, stat) {
+            if (e) return cb(e)
+            else {
+              var buffer = _.allocBuffer(stat.size)
+              fs.read(
+                fd,
+                buffer,
+                0,
+                buffer.length,
+                null,
+                function (e) {
+                  if (e) cb(e)
+                  else cb(null, opts.encoding ? buffer.toString(opts.encoding) : buffer)
+                },
+                '_nested_'
+              )
+            }
+          },
+          '_nested_'
+        )
+      },
+      cb
+    )
+  }
+
+  fs.writeFile = function (path, data, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'w')
+    opts.advice || (opts.advice = 'NOREUSE')
+    _fdOperation(
+      path,
+      opts,
+      function (fd, cb) {
+        if (typeof data === 'string') data = _.bufferFrom(data, opts.encoding || 'utf8')
+        fs.write(
+          fd,
+          data,
+          0,
+          data.length,
+          null,
+          function (e) {
+            cb(e)
+          },
+          '_nested_'
+        )
+      },
+      cb
+    )
+  }
+
+  fs.appendFile = function (path, data, opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
+    }
+    opts.flag || (opts.flag = 'a')
+    fs.writeFile(path, data, opts, cb)
+  }
+
+  fs.truncate = function (path, len, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r+' },
+      function (fd, cb) {
+        fs.ftruncate(fd, len, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.readdir = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r' },
+      function (fd, cb) {
+        fs._readdir(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.mkdir = function (path, mode, cb) {
+    if (typeof mode === 'function') {
+      cb = mode
+      mode = 0777
+    }
+    _fdOperation(
+      path,
+      { flag: '\\wx' },
+      function (fd, cb) {
+        fs._mkdir(fd, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.utimes = function (path, atime, mtime, cb) {
+    _fdOperation(
+      path,
+      { flag: 'r+' },
+      function (fd, cb) {
+        fs.futimes(fd, atime, mtime, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.chmod = fs.lchmod = function (path, mode, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r+' },
+      function (fd, cb) {
+        fs.fchmod(fd, mode, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  fs.chown = fs.lchown = function (path, uid, gid, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r+' },
+      function (fd, cb) {
+        fs.fchown(fd, uid, gid, cb, '_nested_')
+      },
+      cb
+    )
+  }
+
+  /* STUBS */
+
+  fs.link = function (src, dst, cb) {
+    // NOTE: theoretically we _could_ do hard links [with untracked `stat.nlink` count…]
+    _.delayedCall(cb, S.err.NOSYS())
+  }
+
+  fs.symlink = function (src, dst, type, cb) {
+    if (typeof type === 'function') {
+      cb = type
+      type = null
+    }
+    _.delayedCall(cb, S.err.NOSYS())
+  }
+
+  fs.readlink = function (path, cb) {
+    _fdOperation(
+      path,
+      { flag: '\\r' },
+      function (fd, cb) {
+        // the named file is *never* a symbolic link…
+        // NOTE: we still use _fdOperation for catching e.g. NOENT/NOTDIR errors…
+        cb(S.err.INVAL())
+      },
+      cb
+    )
+  }
+
+  fs.realpath = function (path, cache, cb) {
+    if (typeof cache === 'function') {
+      cb = cache
+      cache = null
+    }
+    if (cache)
+      _.delayedCall(cb, S.err.NOSYS()) // TODO: what would be involved here?
+    else
+      _fdOperation(
+        path,
+        { flag: '\\r' },
+        function (fd, cb) {
+          cb(null, _.absolutePath(path))
+        },
+        cb
+      )
+  }
+
+  fs.fchown = function (fd, uid, gid, cb, _n_) {
+    cb = GROUP(
+      cb,
+      function () {
+        var _fd = fileDescriptors[fd]
+        if (!_fd || !_fd.flags.write) _.delayedCall(cb, S.err.BADF())
+        else _.delayedCall(cb, S.err.NOSYS())
+      },
+      _n_ === '_nested_'
+    )
+  }
+
+  // See https://github.com/natevw/fatfs/pull/31/files
+  fs.createLabel = function (name, cb) {
+    fs.open(name, 'a', (e, _fd) => {
+      if (e) {
+        cb(e)
+      } else {
+        let fd = fileDescriptors[_fd]
+        fd.entry.Attr.volume_id = true
+        fd.entry.FstClusLO = 0
+        fd.entry.FstClusHI = 0
+
+        fs._updateEntry(fd.entry, {}, cb)
+      }
+    })
+  }
+
+  return fs
+}

@vates/fatfs/make_sample.sh

@@ -0,0 +1,36 @@
+#! /bin/sh
+
+
+# NOTE: this only works on OS X
+# TODO: Linux could use mkfs stuff without attach?
+
+# ./make_sample.sh /tmp/fat16.img FAT16
+
+FILE=$1
+: ${FILE:?must be provided as first argument}
+
+TYPE=$2
+: ${TYPE:?must be provided as second argument}
+
+SIZE=$3                             # in MB
+case "$TYPE" in
+    "FAT12") : ${SIZE:=1}           # <128
+    ;;
+    "FAT16") : ${SIZE:=5}           # >4
+    ;;
+    "FAT32") : ${SIZE:=33}          # >32
+    ;;
+    *) : ${SIZE:=1}                 # e.g. ExFAT
+esac
+
+echo Making $TYPE of $SIZE MB at $FILE
+
+if [[ "$TYPE" = FAT* ]]
+then
+    TYPE="MS-DOS $TYPE"
+fi
+
+dd if=/dev/zero of=$FILE bs=1048576 count=$SIZE
+DEV=`hdiutil attach $FILE -nomount`
+diskutil eraseVolume "$TYPE" "FATFS TEST" $DEV
+hdiutil detach $DEV

@vates/fatfs/package.json

@@ -0,0 +1,32 @@
+{
+  "private": false,
+  "name": "@vates/fatfs",
+  "version": "0.11.0",
+  "description": "fs implementation on top of raw FAT16/FAT32 block source",
+  "main": "index.js",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra"
+  },
+  "keywords": [
+    "filesystem",
+    "fs",
+    "fat",
+    "fat32",
+    "fat16"
+  ],
+  "author": "Vates SAS",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/vatesfr/xen-orchestra/issues"
+  },
+  "homepage": "https://github.com/vatesfr/xen-orchestra",
+  "dependencies": {
+    "fifolock": "^1.0.0",
+    "struct-fu": "^1.2.1",
+    "xok": "^1.0.0"
+  }
+}

@vates/fatfs/structs.js

@@ -0,0 +1,255 @@
+// see http://staff.washington.edu/dittrich/misc/fatgen103.pdf
+// and http://www.cse.scu.edu/~tschwarz/COEN252_09/Lectures/FAT.html
+
+var _ = require('struct-fu'),
+    __ = require("./helpers.js");
+
+var bootBase = _.struct([
+    _.byte('jmpBoot', 3),
+    _.char('OEMName', 8),
+    _.uint16le('BytsPerSec'),
+    _.uint8('SecPerClus'),
+    _.uint16le('ResvdSecCnt'),      // Rsvd in table, but Resvd in calcs…
+    _.uint8('NumFATs'),
+    _.uint16le('RootEntCnt'),
+    _.uint16le('TotSec16'),
+    _.uint8('Media'),
+    _.uint16le('FATSz16'),
+    _.uint16le('SecPerTrk'),
+    _.uint16le('NumHeads'),
+    _.uint32le('HiddSec'),
+    _.uint32le('TotSec32')
+]);
+
+var bootInfo = _.struct([
+    _.uint8('DrvNum'),
+    _.uint8('Reserved1'),
+    _.uint8('BootSig'),
+    _.uint32le('VolID'),
+    _.char('VolLab', 11),
+    _.char('FilSysType', 8)
+]);
+
+exports.boot16 = _.struct([
+    bootBase,
+    bootInfo
+]);
+
+exports.boot32 = _.struct([
+    bootBase,
+    _.uint32le('FATSz32'),
+    _.struct('ExtFlags', [
+        _.ubit('NumActiveFAT', 4),
+        _.ubit('_reserved1', 3),
+        _.bool('MirroredFAT'),
+        _.ubit('_reserved2', 8)
+    ].reverse()),
+    _.struct('FSVer', [
+        _.uint8('Major'),
+        _.uint8('Minor')
+    ]),
+    _.uint32le('RootClus'),
+    _.uint16le('FSInfo'),
+    _.uint16le('BkBootSec'),
+    _.byte('Reserved', 12),
+    bootInfo
+]);
+
+
+var _time = _.struct([
+    _.ubit('hours',5),
+    _.ubit('minutes',6),
+    _.ubit('seconds_2',5)
+]), time = {
+    valueFromBytes: function (buf, off) {
+        off || (off = {bytes:0});
+        
+        var _buf = __.bufferFrom([buf[off.bytes+1], buf[off.bytes+0]]),
+            val = _time.valueFromBytes(_buf);
+        off.bytes += this.size;
+        return val;
+    },
+    bytesFromValue: function (val, buf, off) {
+        val || (val = {hours:0, minutes:0, seconds_2:0});
+        buf || (buf = __.allocBuffer(this.size));
+        off || (off = {bytes:0});
+        
+        var _buf = _time.bytesFromValue(val);
+        buf[off.bytes+1] = _buf[0];
+        buf[off.bytes+0] = _buf[1];
+        off.bytes += this.size;
+        return buf;
+    },
+    size: _time.size
+};
+
+var _date = _.struct([
+    _.ubit('year',7),
+    _.ubit('month',4),
+    _.ubit('day',5)
+]), date = {
+    valueFromBytes: function (buf, off) {
+        off || (off = {bytes:0});
+        
+        var _buf = __.bufferFrom([buf[off.bytes+1], buf[off.bytes+0]]),
+            val = _date.valueFromBytes(_buf);
+        off.bytes += this.size;
+        return val;
+    },
+    bytesFromValue: function (val, buf, off) {
+        val || (val = {year:0, month:0, day:0});
+        buf || (buf = __.allocBuffer(this.size));
+        off || (off = {bytes:0});
+        
+        var _buf = _date.bytesFromValue(val);
+        buf[off.bytes+1] = _buf[0];
+        buf[off.bytes+0] = _buf[1];
+        off.bytes += this.size;
+        return buf;
+    },
+    size: _date.size
+};
+
+exports.dirEntry = _.struct([
+    _.struct('Name', [
+        _.char('filename',8),
+        _.char('extension',3)
+    ]),
+    _.struct('Attr', [
+        _.bool('readonly'),
+        _.bool('hidden'),
+        _.bool('system'),
+        _.bool('volume_id'),
+        _.bool('directory'),
+        _.bool('archive'),
+        _.ubit('reserved', 2)
+    ].reverse()),
+    _.byte('NTRes', 1),
+    _.uint8('CrtTimeTenth'),
+    _.struct('CrtTime', [time]),
+    _.struct('CrtDate', [date]),
+    _.struct('LstAccDate', [date]),
+    _.uint16le('FstClusHI'),
+    _.struct('WrtTime', [time]),
+    _.struct('WrtDate', [date]),
+    _.uint16le('FstClusLO'),
+    _.uint32le('FileSize')
+]);
+exports.entryDoneFlag = 0x00;
+exports.entryFreeFlag = 0xE5;
+exports.entryIsE5Flag = 0x05;
+
+exports.dirEntry_simple = _.struct([
+    _.struct('Name', [
+        _.char('filename',8),
+        _.char('extension',3)
+    ]),
+    _.padTo(exports.dirEntry.size)
+    /*
+    _.uint8('Attr_raw'),
+    _.byte('NTRes', 1),
+    _.byte('Crt_raw', 1+2+2),
+    _.byte('Lst_raw', 2),
+    _.uint16le('FstClusHI'),
+    _.byte('Wrt_raw', 2+2),
+    _.uint16le('FstClusLO'),
+    _.uint32le('FileSize')
+    */
+]);
+
+
+exports.lastLongFlag = 0x40;
+exports.longDirFlag = 0x0F;
+exports.longDirEntry = _.struct([
+    _.uint8('Ord'),
+    _.char16le('Name1', 10),
+    _.uint8('Attr_raw'),
+    _.uint8('Type'),
+    _.uint8('Chksum'),
+    _.char16le('Name2', 12),
+    _.uint16le('FstClusLO'),
+    _.char16le('Name3', 4)
+]);
+
+if (exports.longDirEntry.size !== exports.dirEntry.size) throw Error("Structs ain't right!");
+
+exports.fatField = {
+    'fat12': _.struct('Status', [
+        _.ubit('field0bc', 8),
+        _.ubit('field1c', 4),
+        _.ubit('field0a', 4),
+        _.ubit('field1ab', 8),
+    ]),
+    'fat16': _.uint16le('Status'),
+    'fat32': _.uint32le('Status')       // more properly this 4 bits reserved + uint28le
+};
+
+exports.fatPrefix = {
+    'fat12': 0xF00,
+    'fat16': 0xFF00,
+    'fat32': 0x0FFFFF00
+};
+
+exports.fatStat = {
+    free: 0x00,
+    _undef: 0x01,
+    rsvMin: 0xF0,
+    bad: 0xF7,
+    eofMin: 0xF8,
+    eof: 0xFF
+};
+
+exports._I = {
+    RUSR: 0400,
+    WUSR: 0200,
+    XUSR: 0100,
+    
+    RGRP: 0040,
+    WGRP: 0020,
+    XGRP: 0010,
+    
+    ROTH: 0004,
+    WOTH: 0002,
+    XOTH: 0001,
+    
+    SUID: 04000,
+    SGID: 02000,
+    SVTX: 01000,
+    
+    FDIR: 040000,
+    FREG: 0100000,
+};
+
+exports._I.RWXU = exports._I.RUSR | exports._I.WUSR | exports._I.XUSR;
+exports._I.RWXG = exports._I.RGRP | exports._I.WGRP | exports._I.XGRP;
+exports._I.RWXO = exports._I.ROTH | exports._I.WOTH | exports._I.XOTH;
+exports._I._sss = exports._I.SUID | exports._I.SGID | exports._I.SVTX;
+exports._I._chmoddable = exports._I.RWXU | exports._I.RWXG | exports._I.RWXO | exports._I._sss;
+
+var _errors = {
+    IO: "Input/output error",
+    NOENT: "No such file or directory",
+    INVAL: "Invalid argument",
+    EXIST: "File exists",
+    NAMETOOLONG: "Filename too long",
+    NOSPC: "No space left on device",
+    NOSYS: "Function not supported",
+    ROFS: "ROFLCopter file system",
+    NOTDIR: "Not a directory",
+    BADF: "Bad file descriptor",
+    EXIST: "File exists",
+    ISDIR: "Is a directory",
+    ACCES: "Permission denied",
+    NOSYS: "Function not implemented",
+    _TODO: "Not implemented yet!"
+};
+
+exports.err = {};
+Object.keys(_errors).forEach(function (sym) {
+    var msg = _errors[sym];
+    exports.err[sym] = function () {
+        var e = new Error(msg);
+        e.code = sym;
+        return e;
+    };
+});

@vates/fatfs/test.js

@@ -0,0 +1,338 @@
+// can be called from CLI with image type, absolute path to an image, or, required as a module
+
+var _ = require("./helpers.js");
+
+var type = process.argv[2];
+if (module.parent) exports.startTests = startTests;
+else if (!type) throw "Usage: node test [FAT12|FAT16|FAT32|ExFAT|…|/path/to/image]";
+else if (type[0] === '/') testWithImage(type);
+else {
+    var uniq = Math.random().toString(36).slice(2),
+        IMG = require('os').tmpdir()+"fatfs-test-"+uniq+".img";
+    require('child_process').exec("./make_sample.sh "+JSON.stringify(IMG)+" "+JSON.stringify(type), function (e,out,err) {
+        if (e) throw e;
+        console.warn(err.toString());
+        //console.log(out.toString());
+        testWithImage(IMG);
+//console.log("open", IMG);
+//return;
+        require('fs').unlink(IMG, function (e) {
+            if (e) console.warn("Error cleaning up test image", e);
+        });
+    });
+}
+
+function testWithImage(imagePath) {
+    var vol = require("./img_volume.js").createDriverSync(imagePath);
+    startTests(vol);
+}
+
+
+function startTests(vol, waitTime) {
+    var fatfs = require("./"),
+        fs = fatfs.createFileSystem(vol, {umask:0020, uid:99, gid:42});
+    
+    waitTime || (waitTime = 0.5e3);
+    
+    [
+        'mkdir','readdir',
+        //'rename','unlink','rmdir',
+        'close','open','fsync',
+        'ftruncate','truncate',
+        'write','read','readFile','writeFile', 'appendFile',
+        
+        'chown','lchown','fchown',
+        'chmod','lchmod', 'fchmod',
+        'utimes','futimes',
+        'stat','lstat','fstat','exists',
+        'link','symlink','readlink','realpath',
+        
+        //'watchFile','unwatchFile','watch'
+    ].forEach(function (method) { assert(method in fs, "fs."+method+" has implementation."); });
+    
+    var BASE_DIR = "/fat_test-"+Math.random().toFixed(20).slice(2),
+        FILENAME = "Simple File.txt",
+        TEXTDATA = "Hello world!";
+    
+    var isReady = false;
+    fs.on('ready', function () {
+        assert(isReady = true, "Driver is ready.");
+    }).on('error', function (e) {
+        assert(e, "If fs driver fires 'error' event, it should include error object.");
+        assert(false, "…but driver should not error when initializing in our case.");
+    });
+    setTimeout(function () {
+        assert(isReady, "Driver fired ready event in timely fashion.");
+    }, waitTime);
+    
+    fs.readdir("/", function (e,files) {
+        assert(isReady, "Method completed after 'ready' event.");
+        assert(!e, "No error reading root directory.");
+        assert(Array.isArray(files), "Got a list of files: "+files);
+    });
+    
+    fs.mkdir(BASE_DIR, function (e) {
+        assert(!e, "No error from fs.mkdir");
+        fs.readdir(BASE_DIR, function (e,arr) {
+            assert(!e, "No error from fs.readdir");
+            assert(arr.length === 0 , "No files in BASE_DIR yet.");
+        });
+        var file = [BASE_DIR,FILENAME].join('/');
+        fs.writeFile(file, TEXTDATA, function (e) {
+            assert(!e, "No error from fs.writeFile");
+            startStreamTests();
+            fs.realpath(file, function (e,path) {
+                assert(!e, "No error from basic fs.realpath call.");
+                assert(path === file, "We already had the real path.");
+            });
+            fs.realpath([BASE_DIR,".","garbage",".","..",FILENAME].join('/'), function (e,path) {
+                assert(!e, "No error from fluffy fs.realpath call.");
+                assert(path === file, "Fixed fluffy path matches normal one.");
+            });
+            fs.realpath([BASE_DIR,"non","existent","path"].join('/'), function (e) {
+                assert(e, "Expected error calling fs.realpath on non-existent file.");
+            });
+            
+            fs.readdir(BASE_DIR, function (e, arr) {
+                assert(!e, "Still no error from fs.readdir");
+                assert(arr.length === 2, "Test directory contains two files.");     // (ours + startStreamTests's)
+                assert(arr[0] === FILENAME, "Filename is correct.");
+                
+                fs.stat(file, function (e,d) {
+                    assert(!e, "No error from fs.stat");
+                    assert(d.isFile() === true, "Result is a file…");
+                    assert(d.isDirectory() === false, "…and not a directory.");
+                    assert(d.size === Buffer.byteLength(TEXTDATA), "Size matches length of content written.");
+                });
+                fs.exists(file, function (bool) {
+                  assert(bool === true, "File exists.");
+                });
+                fs.readFile(file, {encoding:'utf8'}, function (e, d) {
+                    assert(!e, "No error from fs.readFile");
+                    assert(d === TEXTDATA, "Data matches what was written.");
+                });
+                // now, overwrite the same file and make sure that goes well too
+                fs.writeFile(file, _.bufferFrom([0x42]), function (e) {
+                    assert(!e, "Still no error from fs.writeFile");
+                    fs.readdir(BASE_DIR, function (e, arr) {
+                        assert(!e, "No error from fs.readdir");
+                        assert(arr.length === 2, "Test directory still contains two files.");
+                        assert(arr[0] === FILENAME, "Filename still correct.");
+                        fs.stat(file, function (e,d) {
+                            assert(!e, "Still no error from fs.stat");
+                            assert(d.isFile() === true, "Result is still a file…");
+                            assert(d.isDirectory() === false, "…and not a directory.");
+                            assert(d.size === 1, "Size matches length of now-truncated content.");
+                        });
+                        fs.readFile(file, function (e, d) {
+                            assert(!e, "Still no error from fs.readFile");
+                            assert(Buffer.isBuffer(d), "Result without encoding is a buffer.");
+                            assert(d.length === 1, "Buffer is correct size.");
+                            assert(d[0] === 0x42, "Buffer content is correct.");
+                            
+                            fs.truncate(file, 1025, function (e) {
+                                assert(!e, "No error from fs.truncate (extending)");
+                                fs.readFile(file, function (e, d) {
+                                    assert(!e, "Still no error from fs.readFile after extension");
+                                    assert(d.length === 1025, "Read after extension is correct size.");
+                                    assert(d[0] === 0x42, "First byte is still correct.");
+                                    var allZeroes = true;
+                                    for (var i = 1, len = d.length; i < len; ++i) if (d[i] !== 0) allZeroes = false;
+                                    assert(allZeroes, "Extended portion of file is zero-filled.");
+                                    
+                                    fs.truncate(file, 3, function (e) {
+                                        assert(!e, "No error from fs.truncate (shortening)");
+                                        fs.readFile(file, function (e, d) {
+                                            assert(!e, "Still no error from fs.readFile after shortening.");
+                                            assert(d.length === 3, "Read after shortening is correct size.");
+                                            assert(d[0] === 0x42, "First byte is still correct.");
+                                            assert(d[1] === 0x00, "Second byte is still correct.");
+                                            assert(d[2] === 0x00, "Third byte is still correct.");
+                                            proceedWithMoreTests();
+                                        });
+                                    }); 
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        });
+        
+        
+        function proceedWithMoreTests() {
+            var fd;
+            fs.open(file, 'r', function (e, _fd) {
+                assert(!e, "No error from fs.open.");
+                fd = _fd;
+            });
+            var was = "\u0042\u0000\u0000",
+                str = "abc";
+            fs.appendFile(file, str, function (e) {
+                assert(!e, "No error from fs.appendFile.");
+                assert(fd, "File descriptor opened before appendFile called.");
+                var buf = _.allocBuffer(str.length);
+                fs.read(fd, buf, 0, buf.length, was.length, function (e,n,d) {
+                    assert(!e, "No error from fs.read after append.");
+                    assert(n === str.length, "All appended data was readable.");
+                    assert(d === buf, "Buffer returned from fs.read matched what was passed in.");
+                    assert(d.toString() === str, "Correct data found where expected.");
+                });
+            });
+            fs.readFile(file, {encoding:'ascii'}, function (e,d) {
+                assert(!e, "No error from fs.appendFile.");
+                assert(d.length === 6, "Read is correct size after append.");
+                assert(d === was+str, "Read string matches what was written and then appended.");
+                
+                fs.open(file, 'a', function (e, fd2) {
+                    assert(!e, "No error from second open of file.");
+                    var str2 = "zyx",
+                        buf2 = _.allocBuffer(str2.length+2);
+                    buf2.write(str2, 1);
+                    fs.write(fd2, buf2, 1, buf2.length-2, was.length, function (e,n,d) {
+                        assert(!e, "No error from appending fs.write.");
+                        assert(n === buf2.length-2, "Wrote proper amount from buffer.");
+                        assert(d === buf2, "Returned original buffer.");
+                        
+                        buf2.fill(0);
+                        buf2[0] = 0xFF;
+                        fs.read(fd, buf2, 1, buf2.length-1, was.length, function (e,n,d) {
+                            assert(!e, "No error from twice-appended read.");
+                            assert(n === buf2.length-1, "Read proper amount into buffer.");
+                            assert(buf2[0] === 0xFF, "Read left first byte in buffer properly alone.");
+                            assert(d.slice(1).toString() === (str+str2).slice(0, buf2.length-1), "Data was appended, not written at position.");
+                        });
+                    });
+                });
+            });
+            
+            var F = [BASE_DIR,"Manually inspect from time to time, please!.txt"].join('/'),
+                S = 512,
+                N = 16,
+                b = _.allocBuffer(S*N);
+            for (var i = 0; i < N; ++i) b.slice(S*i, S*i+S).fill(i.toString(16).charCodeAt(0));
+            fs.writeFile(F, b, function (e) {
+                assert(!e, "No error from fs.writeFile with counting blocks.");
+                fs.readFile(F, function (e,d) {
+                    assert(!e, "No error from fs.readFile with counting blocks.");
+                    assert(d.length === b.length, "Readback is correct size");
+                    var matched = true;
+                    for (var i = 0; i < S*N; ++i) if (b[i] !== d[i]) matched = false;
+                    assert(matched, "Readback matches write byte-for-byte");
+                });
+            });
+            
+            fs.stat(F, function (e,d) {
+                assert(!e, "No error from fs.stat on counting blocks file.");
+                assert(d.atime instanceof Date && !isNaN(d.atime.getTime()), "Access time is a valid date.");
+                assert(d.mtime instanceof Date && !isNaN(d.mtime.getTime()), "Modify time is a valid date.");
+                assert(d.ctime instanceof Date && !isNaN(d.ctime.getTime()), "Change^WCreate time is a valid date.");
+                var tf = d.ctime.getTime(),
+                    ct = Date.now();
+                assert(tf - 2*waitTime < ct && ct < tf + 2*waitTime, "Create time is within ± two `waitTime`s of now.");
+                fs.utimes(F, new Date(2009, 7-1, 2), null, function (e) {
+                    assert(!e, "No error from fs.utimes.");
+                    fs.stat(F, function (e,d2) {
+                        assert(!e, "No fs.stat error after touching timestamps.");
+                        assert(+d.ctime === +d2.ctime, "Create time not changed by fs.utimes");
+                        assert(d2.atime.toString().indexOf("Jul 02 2009 00:00:00") === 4, "Access time set correctly");
+                        var tf = d2.mtime.getTime(),
+                            ct = Date.now();
+                        assert(tf-2e3 < ct && ct < tf+2e3+waitTime, "Modify time is within ± a few seconds of now.");
+                        
+                        // NOTE: due to serialization, this can check results of the `fs.chmod` below, too!
+                        assert(!(d2.mode & 0100), "Archive bit is now unset.");
+                        assert(!(d2.mode & 0222), "Writable perms are unset.");
+                        assert(d2.mode & 0100000, "Regular file bit is set.");
+                    });
+                });
+                
+                assert(d.uid === 99, "Desired UID applied.");
+                assert(d.gid === 42, "Desired GID applied.");
+                assert(d.mode & 0100, "Archive bit is set.");
+                assert(d.mode & 0200, "Writable perm is set for user.");
+                assert((d.mode & 0022) === 0002, "Writable perm is only masked out for group.");
+                fs.chmod(F, 0422, function (e) {
+                    assert(!e, "No error from fs.chmod.");
+                });
+            });
+            
+            fs.chown(F, 99, 256, function (e) {
+                assert(e && e.code === 'NOSYS', "Expected error from fs.fchown.");
+            });
+        }
+        
+        function startStreamTests() {
+            var file2 = [BASE_DIR,FILENAME+"2"].join('/'),
+                outStream = fs.createWriteStream(file2);
+            var outStreamOpened = false;
+            outStream.on('open', function (fd) {
+                outStreamOpened = true;
+                assert(typeof fd === 'number', "Got file descriptor on fs.createWriteStream open.");
+            }).on('error', function (e) {
+                assert(e, "If fs.createWriteStream fires 'error' event, it should include error object.");
+                assert(false, "But, fs.createWriteStream should not error during these tests.");
+            });
+            setTimeout(function () {
+                assert(outStreamOpened, "outStream fired 'open' event in a timely fashion.");
+            }, waitTime);
+            var TEXT_MOD = TEXTDATA.toLowerCase()+"\n",
+                NUM_REPS = (waitTime <= 1e3) ? 1024 : 16;
+            outStream.write(TEXT_MOD, 'utf16le');
+            outStream.write("Ο καλύτερος χρόνος να φυτευτεί ένα \ud83c\udf31 είναι δέκα έτη πριν.", 'utf16le');
+            outStream.write("La vez del segundo mejor ahora está.\n", 'utf16le');
+            for (var i = 0; i < NUM_REPS; ++i) outStream.write("123456789\n", 'ascii');
+            outStream.write("JavaScript how do they work\n", 'utf16le');
+            outStream.write("The end, almost.\n", 'utf16le');
+            outStream.end(TEXTDATA, 'utf16le');
+            var outStreamFinished = false;
+            outStream.on('finish', function () {
+                outStreamFinished = true;
+                
+                var inStream = fs.createReadStream(file2, {start:NUM_REPS*10, encoding:'utf16le', autoClose:false}),
+                    gotData = false, gotEOF = false, inStreamFD = null;
+                inStream.on('open', function (fd) {
+                    assert(typeof fd === 'number', "Got file descriptor on fs.createReadStream open.");
+                    inStreamFD = fd;
+                });
+                inStream.on('data', function (d) {
+                    gotData = true;
+                    assert(typeof d === 'string', "Data returned as string.");
+                    assert(d.slice(d.length-TEXTDATA.length) === TEXTDATA, "End of file matches what was written.");
+                });
+                inStream.on('end', function () {
+                    gotEOF = true;
+                    
+                    var len = Buffer.byteLength(TEXT_MOD, 'utf16le'),
+                        buf = _.allocBuffer(len);
+                    fs.fsync(inStreamFD, function (e) {
+                        assert(!e, "No error from proper fsync.");
+                    });
+                    fs.fsync('garbage', function (e) {
+                        assert(e, "Expected error from garbage fsync.");
+                    });
+                    fs.read(inStreamFD, buf, 0, len, 0, function (e,n,d) {
+                        assert(!e, "No error reading from beginning of inStream's file descriptor.");
+                        assert(n === len, "Read complete buffer at beginning of inStream's fd.");
+                        assert(d.toString('utf16le') === TEXT_MOD, "Data matches at beginning of inStream's fd.");
+                        fs.close(inStreamFD, function (e) {
+                            assert(!e, "No error closing inStream's fd.");
+                        });
+                    });
+                });
+                setTimeout(function () {
+                    assert(gotData, "inStream fired 'data' event in a timely fashion.");
+                    setTimeout(function () {
+                        assert(gotEOF, "inStream fired 'eof' event in a timely fashion.");
+                    }, waitTime);
+                }, waitTime);
+            });
+            setTimeout(function () {
+                assert(outStreamFinished, "outStream fired 'finish' event in a timely fashion.");
+            }, 2*waitTime);
+        }
+    });
+}
+
+
+function assert(b,msg) { if (!msg) console.warn("no msg", Error().stack); if (!b) throw Error("Assertion failure. "+msg); else console.log(msg); }

@vates/fatfs/vol.js

@@ -0,0 +1,165 @@
+var S = require("./structs.js"),
+    c = require("./chains.js"),
+    $ = require("./cache.js"),
+    _ = require("./helpers.js");
+
+exports.init = function (volume, opts, bootSector) {
+    if (bootSector[510] !== 0x55 || bootSector[511] !== 0xAA) throw Error("Invalid volume signature!");
+    
+    var isFAT16 = bootSector.readUInt16LE(S.boot16.fields['FATSz16'].offset),
+        bootStruct = (isFAT16) ? S.boot16 : S.boot32,
+        BS = bootStruct.valueFromBytes(bootSector);
+    _.log(_.log.DBG, "Boot sector info:", BS);
+    bootSector = null;      // allow GC
+    if (!BS.BytsPerSec) throw Error("This looks like an ExFAT volume! (unsupported)");
+    else if (BS.BytsPerSec !== volume.sectorSize) throw Error("Sector size mismatch with FAT table.");
+    
+    var FATSz = (isFAT16) ? BS.FATSz16 : BS.FATSz32,
+        rootDirSectors = Math.ceil((BS.RootEntCnt * 32) / BS.BytsPerSec),
+        firstDataSector = BS.ResvdSecCnt + (BS.NumFATs * FATSz) + rootDirSectors,
+        totSec = (BS.TotSec16) ? BS.TotSec16 : BS.TotSec32,
+        dataSec = totSec - firstDataSector,
+        countofClusters = Math.floor(dataSec / BS.SecPerClus);
+    // avoid corrupting sectors from other partitions or whatnot
+    if (totSec > volume.numSectors) throw Error("Volume size mismatch!");
+    
+    var fatType;
+    if (countofClusters < 4085) {
+        fatType = 'fat12';
+    } else if (countofClusters < 65525) {
+        fatType = 'fat16';
+    } else {
+        fatType = 'fat32';
+    }
+    
+    _.log(_.log.DBG, "rootDirSectors", rootDirSectors, "firstDataSector", firstDataSector, "countofClusters", countofClusters, "=>", fatType);
+    
+    var vol = {};
+    
+    vol.opts = opts;
+    
+    vol._sectorSize = BS.BytsPerSec;
+    vol._sectorsPerCluster = BS.SecPerClus;
+    vol._firstSectorOfCluster = function (n) {
+        return firstDataSector + (n-2)*vol._sectorsPerCluster;
+    };
+    
+    vol._makeCache = function () {
+        return $.wrapDriver(volume);
+    };
+    
+    vol._readSectors = function (cache, secNum, dest, cb) {
+        if (typeof dest === 'function') {
+            cb = dest;
+            dest = _.allocBuffer(vol._sectorSize);
+        }
+        _.log(_.log.DBG, "vol._readSectors", secNum, dest.length);
+        if (secNum < volume.numSectors) cache.readSectors(secNum, dest, function (e) { cb(e, dest); });
+        else throw Error("Invalid sector number!");
+    };
+    
+    vol._writeSectors = function (cache, secNum, data, cb) {
+        _.log(_.log.DBG, "vol._writeSectors", secNum, data.length);
+        // NOTE: these are internal assertions, public API will get proper `S.err`s
+        if (data.length % volume.sectorSize) throw Error("Buffer length not a multiple of sector size");
+        else if (opts.ro) throw Error("Read-only filesystem");
+        else if (secNum < volume.numSectors) cache.writeSectors(secNum, data, cb);
+        else throw Error("Invalid sector number!");
+    };
+    
+    function fatInfoForCluster(n) {
+        var entryStruct = S.fatField[fatType],
+            FATOffset = (fatType === 'fat12') ? Math.floor(n/2) * entryStruct.size : n * entryStruct.size,
+            SecNum = BS.ResvdSecCnt + Math.floor(FATOffset / BS.BytsPerSec);
+            EntOffset = FATOffset % BS.BytsPerSec;
+        return {sector:SecNum-BS.ResvdSecCnt, offset:EntOffset, struct:entryStruct};
+    }
+    
+    // TODO: all this FAT manipulation is crazy inefficient! needs read caching *and* write caching
+    //        …the best place for cache might be in `volume` handler, though. add a `flush` method to that spec?
+    // TODO: how should we handle redundant FATs? mirror every write? just ignore completely? copy-on-eject?
+    
+    var fatChain = c.sectorChain(vol, BS.ResvdSecCnt, FATSz);
+    fatChain.cacheAdvice = 'RANDOM';
+    
+    vol.fetchFromFAT = function (clusterNum, cb) {
+        var info = fatInfoForCluster(clusterNum);
+        fatChain.readFromPosition(info, info.struct.size, function (e,n,d) {
+            if (e) return cb(e);
+            var status = info.struct.valueFromBytes(d), prefix;
+            if (fatType === 'fat12') {
+                if (clusterNum % 2) {
+                    status = (status.field1ab << 4) + status.field1c;
+                } else {
+                    status = (status.field0a << 8) + status.field0bc;
+                }
+            }
+            else if (fatType === 'fat32') {
+                status &= 0x0FFFFFFF;
+            }
+            
+            var prefix = S.fatPrefix[fatType];
+            if (status === S.fatStat.free) cb(null, 'free');
+            else if (status === S.fatStat._undef) cb(null, '-invalid-');
+            else if (status > prefix+S.fatStat.eofMin) cb(null, 'eof');
+            else if (status === prefix+S.fatStat.bad) cb(null, 'bad');
+            else if (status > prefix+S.fatStat.rsvMin) cb(null, 'reserved');
+            else cb(null, status);
+        });
+    };
+    
+    vol.storeToFAT = function (clusterNum, status, cb) {
+        if (typeof status === 'string') {
+            status = S.fatStat[status];
+            status += S.fatPrefix[fatType];
+        }
+        var info = fatInfoForCluster(clusterNum);
+        // TODO: technically fat32 needs to *preserve* the high 4 bits
+        if (fatType === 'fat12') fatChain.readFromPosition(info, info.struct.size, function (e,n,d) {
+            var value = info.struct.valueFromBytes(d);
+            if (clusterNum % 2) {
+                value.field1ab = status >>> 4;
+                value.field1c = status & 0x0F;
+            } else {
+                value.field0a = status >>> 8;
+                value.field0bc = status & 0xFF;
+            }
+            var entry = info.struct.bytesFromValue(value);
+            fatChain.writeToPosition(info, entry, cb);
+        }); else {
+            var entry = info.struct.bytesFromValue(status);
+            fatChain.writeToPosition(info, entry, cb);
+        }
+    };
+    
+    vol.allocateInFAT = function (hint, cb) {
+        if (typeof hint === 'function') {
+            cb = hint;
+            hint = 2;   // TODO: cache a better starting point?
+        }
+        function searchForFreeCluster(num, cb) {
+            if (num < countofClusters) vol.fetchFromFAT(num, function (e, status) {
+                if (e) cb(e);
+                else if (status === 'free') cb(null, num);
+                else searchForFreeCluster(num+1, cb);
+            }); else cb(S.err.NOSPC());     // TODO: try searching backwards from hint…
+        }
+        searchForFreeCluster(hint, function (e, clusterNum) {
+            if (e) cb(e);
+            else vol.storeToFAT(clusterNum, 'eof', cb.bind(null,null,clusterNum));
+        });
+    };
+    
+    vol.rootDirectoryChain = (isFAT16) ?
+        c.sectorChain(vol, firstDataSector - rootDirSectors, rootDirSectors) :
+        c.clusterChain(vol, BS.RootClus);
+    vol.rootDirectoryChain.cacheAdvice = 'WILLNEED';
+    vol.chainForCluster = c.clusterChain.bind(c, vol);
+    vol.chainFromJSON = function (d) {
+        return ('numSectors' in d) ?
+            c.sectorChain(vol, d.firstSector, d.numSectors) :
+            c.clusterChain(vol, d.firstCluster);
+    };
+    
+    return vol;
+}

@vates/fuse-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/fuse-vhd",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "license": "ISC",
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,12 +19,12 @@
   },
   "main": "./index.mjs",
   "dependencies": {
-    "@xen-orchestra/fs": "^4.1.7",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/fs": "^4.3.0",
+    "@xen-orchestra/log": "^0.7.1",
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.11.0"
+    "vhd-lib": "^4.11.2"
   },
   "bin": {
     "xo-fuse-vhd": "./cli.mjs"

@vates/json-hash/index.test.js

@@ -2,7 +2,7 @@
 
 const { jsonHash } = require('@vates/json-hash')
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 const cases = [
   [null, 'dCNOmK_nSY-12vHzasLXiswzlGT5UHA7jAGYkvmCuQs'],

@vates/json-hash/package.json

@@ -23,11 +23,8 @@
   "engines": {
     "node": ">=14.18.0"
   },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   }
 }

@vates/multi-key-map/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('node:assert')
 
 const { MultiKeyMap } = require('./')

@vates/multi-key-map/package.json

@@ -24,9 +24,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.2.1"
+    "test": "node --test"
   }
 }

@vates/nbd-client/multi.mjs

@@ -34,9 +34,8 @@ export default class MultiNbdClient {
   async connect() {
     const candidates = [...this.#settings]
 
-    const promises = []
     const baseOptions = this.#options
-    const  _connect = async () => {
+    const _connect = async () => {
       if (candidates.length === 0) {
         return
       }
@@ -63,11 +62,11 @@ export default class MultiNbdClient {
         return _connect()
       }
     }
+    // don't connect in parallel since this can lead to race condition
+    // on distributed systems ( like the NBD server of the XAPI)
     for (let i = 0; i < this.#nbdConcurrency; i++) {
-      promises.push(_connect())
+      await _connect()
     }
-    await Promise.all(promises)
-
     if (this.#clients.length === 0) {
       throw new Error(`Fail to connect to any Nbd client`)
     }

@vates/nbd-client/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "3.1.0",
+  "version": "3.1.2",
   "engines": {
     "node": ">=14.0"
   },
@@ -22,9 +22,9 @@
     "@vates/async-each": "^1.0.0",
     "@vates/read-chunk": "^1.2.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/log": "^0.7.1",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^4.2.0"
+    "xen-api": "^4.7.0"
   },
   "devDependencies": {
     "tap": "^18.7.0",

@vates/node-vsphere-soap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/node-vsphere-soap",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
   "main": "lib/client.mjs",
   "author": "reedog117",
@@ -11,14 +11,11 @@
   },
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/log": "^0.7.1",
     "axios": "^1.4.0",
     "soap": "^1.0.0",
     "soap-cookie": "^0.10.1"
   },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
   "keywords": [
     "vsphere",
     "vcenter",

@vates/node-vsphere-soap/test/vsphere-soap.test.mjs

@@ -5,7 +5,7 @@
 */
 
 import assert from 'assert'
-import { describe, it } from 'test'
+import { describe, it } from 'node:test'
 
 import * as vc from '../lib/client.mjs'
 

@vates/otp/index.test.mjs

@@ -1,5 +1,5 @@
 import { strict as assert } from 'node:assert'
-import test from 'test'
+import test from 'node:test'
 
 import {
   generateHotp,

@vates/otp/package.json

@@ -31,9 +31,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
+    "test": "node --test"
   }
 }

@vates/parse-duration/index.js

@@ -6,9 +6,11 @@ exports.parseDuration = value => {
   if (typeof value === 'number') {
     return value
   }
-  const duration = ms(value)
-  if (duration === undefined) {
-    throw new TypeError(`not a valid duration: ${value}`)
+  if (typeof value === 'string' && value !== '') {
+    const duration = ms(value)
+    if (duration !== undefined) {
+      return duration
+    }
   }
-  return duration
+  throw new TypeError(`not a valid duration: ${value}`)
 }

@vates/parse-duration/index.test.mjs

@@ -0,0 +1,22 @@
+import { describe, it } from 'node:test'
+import { parseDuration } from '@vates/parse-duration'
+import assert from 'node:assert/strict'
+
+describe('parseDuration()', () => {
+  it('should parse string', () => {
+    const input = '2 days'
+    const expected = 172800000
+    assert.strictEqual(parseDuration(input), expected)
+  })
+
+  it('should return its input if already a number', () => {
+    const input = 172800000
+    assert.strictEqual(parseDuration(input), input)
+  })
+
+  for (const input of [undefined, '', 'invalid duration']) {
+    it('should throw an error for ' + input, () => {
+      assert.throws(() => parseDuration(input), { message: `not a valid duration: ${input}` })
+    })
+  }
+})

@vates/parse-duration/package.json

@@ -27,6 +27,7 @@
     "ms": "^2.1.2"
   },
   "scripts": {
-    "postversion": "npm publish --access public"
+    "postversion": "npm publish --access public",
+    "test": "node --test"
   }
 }

@vates/predicates/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 
 const { every, not, some } = require('./')
 

@vates/predicates/package.json

@@ -32,9 +32,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
+    "test": "node --test"
   }
 }

@vates/read-chunk/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('node:assert').strict
 
 const { Readable } = require('stream')

@vates/read-chunk/package.json

@@ -25,15 +25,12 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   },
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
   },
-  "devDependencies": {
-    "test": "^3.2.1"
-  },
   "dependencies": {
     "isutf8": "^4.0.0"
   }

@vates/stream-reader/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('node:assert').strict
 
 const { Readable } = require('stream')

@vates/stream-reader/package.json

@@ -31,9 +31,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
+    "test": "node --test"
   }
 }

@vates/task/.USAGE.md

@@ -49,6 +49,15 @@ const result = await task.runInside(fn)
 // if fn rejects, the task will be marked as failed
 // if fn resolves, the task will be marked as succeeded
 const result = await task.run(fn)
+
+// manually starts the task
+task.start()
+
+// manually finishes the task in success state
+task.success(result)
+
+// manually finishes the task in failure state
+task.failure(error)
 ```
 
 Inside a task:

@vates/task/README.md

@@ -65,6 +65,15 @@ const result = await task.runInside(fn)
 // if fn rejects, the task will be marked as failed
 // if fn resolves, the task will be marked as succeeded
 const result = await task.run(fn)
+
+// manually starts the task
+task.start()
+
+// manually finishes the task in success state
+task.success(result)
+
+// manually finishes the task in failure state
+task.failure(error)
 ```
 
 Inside a task:

@vates/task/combineEvents.js

@@ -44,6 +44,8 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
       } else if (type === 'property') {
         ;(taskLog.properties ?? (taskLog.properties = { __proto__: null }))[event.name] = event.value
       } else if (type === 'end') {
+        taskLogs.delete(id)
+
         taskLog.end = event.timestamp
         taskLog.result = event.result
         taskLog.status = event.status

@vates/task/combineEvents.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert').strict
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 
 const { makeOnProgress } = require('./combineEvents.js')
 const { Task } = require('./index.js')

@vates/task/index.js

@@ -131,10 +131,15 @@ class Task {
 
   #end(status, result) {
     assert.equal(this.#status, PENDING)
+    assert.equal(this.#running, false)
 
-    this.#status = status
     this.#emit('end', { status, result })
     this.#onProgress = alreadyEnded
+    this.#status = status
+  }
+
+  failure(error) {
+    this.#end(FAILURE, error)
   }
 
   info(message, data) {
@@ -142,10 +147,8 @@ class Task {
   }
 
   #maybeStart() {
-    const startData = this.#startData
-    if (startData !== undefined) {
-      this.#startData = undefined
-      this.#emit('start', startData)
+    if (this.#startData !== undefined) {
+      this.start()
     }
   }
 
@@ -170,6 +173,7 @@ class Task {
       this.#running = false
       return result
     } catch (result) {
+      this.#running = false
       this.#end(FAILURE, result)
       throw result
     }
@@ -181,6 +185,18 @@ class Task {
     this.#emit('property', { name, value })
   }
 
+  start() {
+    const startData = this.#startData
+    assert.notEqual(startData, undefined, 'task has already started')
+
+    this.#startData = undefined
+    this.#emit('start', startData)
+  }
+
+  success(result) {
+    this.#end(SUCCESS, result)
+  }
+
   warning(message, data) {
     assert.equal(this.status, PENDING)
 

@vates/task/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert').strict
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 
 const { makeOnProgress } = require('./combineEvents.js')
 const { Task } = require('./index.js')
@@ -145,6 +145,43 @@ describe('Task', function () {
     })
   })
 
+  describe('#failure()', function () {
+    const error = new Error()
+
+    it('throws if the task has not started yet', function () {
+      const task = createTask()
+
+      assert.throws(() => task.failure(error), { message: /^task has not started yet\b/ })
+    })
+
+    it('throws if the task is running', function () {
+      const task = createTask()
+      task.runInside(noop)
+
+      assert.throws(() => task.failure(error), { code: 'ERR_ASSERTION' })
+    })
+
+    it('throws if the task has already finished', function () {
+      const task = createTask()
+      task.start()
+      task.success()
+
+      assert.throws(() => task.failure(error), { code: 'ERR_ASSERTION' })
+    })
+
+    it('finishes the task and mark it as failed', function () {
+      const task = createTask()
+      task.start()
+      task.failure(error)
+
+      assertEvent(task, {
+        status: 'failure',
+        result: error,
+        type: 'end',
+      })
+    })
+  })
+
   describe('.info()', function () {
     it('does nothing when run outside a task', function () {
       Task.info('foo')
@@ -225,7 +262,7 @@ describe('Task', function () {
 
     it('throws when the task is not started', function () {
       const task = createTask()
-      assert.throws(() => task.set(name, value), { message: 'task has not started yet' })
+      assert.throws(() => task.set(name, value), { message: /^task has not started yet\b/ })
     })
 
     it(`emits an property message`, async function () {
@@ -244,6 +281,59 @@ describe('Task', function () {
     })
   })
 
+  describe('#start', function () {
+    it('starts the task', function () {
+      const task = createTask()
+      task.start()
+
+      assertEvent(task, { type: 'start' })
+    })
+
+    it('throws when the task has already started', function () {
+      const task = createTask()
+      task.start()
+
+      assert.throws(() => task.start(), { message: 'task has already started' })
+    })
+  })
+
+  describe('#success()', function () {
+    const result = {}
+
+    it('throws if the task has not started yet', function () {
+      const task = createTask()
+
+      assert.throws(() => task.success(result), { message: /^task has not started yet\b/ })
+    })
+
+    it('throws if the task is running', function () {
+      const task = createTask()
+      task.runInside(noop)
+
+      assert.throws(() => task.success(result), { code: 'ERR_ASSERTION' })
+    })
+
+    it('throws if the task has already finished', function () {
+      const task = createTask()
+      task.start()
+      task.success()
+
+      assert.throws(() => task.success(result), { code: 'ERR_ASSERTION' })
+    })
+
+    it('finishes the task and mark it as successful', function () {
+      const task = createTask()
+      task.start()
+      task.success(result)
+
+      assertEvent(task, {
+        status: 'success',
+        result,
+        type: 'end',
+      })
+    })
+  })
+
   describe('.warning()', function () {
     it('does nothing when run outside a task', function () {
       Task.warning('foo')
@@ -266,7 +356,7 @@ describe('Task', function () {
     describe(`#${type}()`, function () {
       it('throws when the task is not started', function () {
         const task = createTask()
-        assert.throws(() => task[type]('foo'), { message: 'task has not started yet' })
+        assert.throws(() => task[type]('foo'), { message: /^task has not started yet\b/ })
       })
 
       it(`emits an ${type} message`, async function () {

@vates/task/package.json

@@ -13,16 +13,13 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "engines": {
     "node": ">=14"
   },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   },
   "exports": {
     ".": "./index.js",

@vates/xml-rpc/.USAGE.md

@@ -121,7 +121,8 @@ const xml = `
 </methodResponse>
 `
 
-const tree = parseXml(xml)
+// do not normalize or trim whitespaces as it will break string values
+const tree = parseXml(xml, { normalize: false, trim: false })
 
 // it can now be passed to the XML-RPC parser
 xmlRpcParser.parse(tree)

@vates/xml-rpc/README.md

@@ -139,7 +139,8 @@ const xml = `
 </methodResponse>
 `
 
-const tree = parseXml(xml)
+// do not normalize or trim whitespaces as it will break string values
+const tree = parseXml(xml, { normalize: false, trim: false })
 
 // it can now be passed to the XML-RPC parser
 xmlRpcParser.parse(tree)

@vates/xml-rpc/package.json

@@ -19,8 +19,7 @@
     "node": ">=12"
   },
   "devDependencies": {
-    "@vates/xml": "^2.0.0",
-    "test": "^3.3.0"
+    "@vates/xml": "^2.0.0"
   },
   "exports": {
     "./formatter": "./formatter.js",
@@ -28,6 +27,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   }
 }

@vates/xml-rpc/test/formatter.js

@@ -6,7 +6,7 @@ const { parseXml } = require('@vates/xml/parse')
 const { readFileSync } = require('node:fs')
 const { xmlRpcFormatter } = require('@vates/xml-rpc/formatter')
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 const methodCall = require('./methodCall.js')
 

@vates/xml-rpc/test/parser.js

@@ -5,7 +5,7 @@ const { parseXml } = require('@vates/xml/parse')
 const { readFileSync } = require('node:fs')
 const { xmlRpcParser } = require('@vates/xml-rpc/parser')
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 const methodCall = require('./methodCall.js')
 

@vates/xml/package.json

@@ -21,15 +21,12 @@
   "dependencies": {
     "sax": "^1.3.0"
   },
-  "devDependencies": {
-    "test": "^3.3.0"
-  },
   "exports": {
     "./format": "./format.js",
     "./parse": "./parse.js"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   }
 }

@vates/xml/test/format.js

@@ -4,7 +4,7 @@ const { formatXml } = require('@vates/xml/format')
 const { join } = require('node:path')
 const { readFileSync } = require('node:fs')
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 test('format()', function () {
   const xml = String(readFileSync(join(__dirname, 'data.xml')))

@vates/xml/test/parse.js

@@ -4,7 +4,7 @@ const { parseXml } = require('@vates/xml/parse')
 const { readFileSync } = require('node:fs')
 const { join } = require('node:path')
 const assert = require('node:assert/strict')
-const test = require('test')
+const test = require('node:test')
 
 test('parse()', function () {
   const xml = String(readFileSync(join(__dirname, 'data.xml')))

@xen-orchestra/async-map/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('assert').strict
 const sinon = require('sinon')
 

@xen-orchestra/async-map/package.json

@@ -32,10 +32,9 @@
   },
   "scripts": {
     "postversion": "npm publish",
-    "test": "node--test"
+    "test": "node --test"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
-    "test": "^3.2.1"
+    "sinon": "^18.0.0"
   }
 }

@xen-orchestra/audit-core/index.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('assert/strict')
-const { afterEach, describe, it } = require('test')
+const { afterEach, describe, it } = require('node:test')
 
 const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')
 

@xen-orchestra/audit-core/package.json

@@ -7,17 +7,17 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.3.0",
+  "version": "0.3.1",
   "engines": {
     "node": ">=14"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test": "node--test"
+    "test": "node --test"
   },
   "dependencies": {
     "@vates/decorate-with": "^2.1.0",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/log": "^0.7.1",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"
   },
@@ -26,8 +26,5 @@
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
-  },
-  "devDependencies": {
-    "test": "^3.3.0"
   }
 }

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.52.2",
-    "@xen-orchestra/fs": "^4.1.7",
+    "@xen-orchestra/backups": "^0.57.0",
+    "@xen-orchestra/fs": "^4.3.0",
     "filenamify": "^6.0.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.0.22",
+  "version": "1.0.28",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/.USAGE.md

@@ -0,0 +1,10 @@
+## Changing the default backup configuration
+
+This library receives a backup configuration from `xo-server` or `xo-proxy`. This configuration is split between multiple properties:
+
+- `config` contains values related to the general backup configuration
+- `config.defaultSettings`, `config.vm.defaultSettings` and `config.metadata.defaultSettings` contain values related to backup jobs default settings:
+  - `config.vm.defaultSettings` is related to VM backup jobs (see [DEFAULT_XAPI_VM_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/VmsRemote.mjs#L15))
+  - `config.metadata.defaultSettings` is related to metadata backup jobs (see [DEFAULT_METADATA_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/Metadata.mjs#L12))
+  - `config.defaultSettings` is related to all backup jobs (see [DEFAULT_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/_Abstract.mjs#L7))
+  - in case of duplicate value, `config.vm.defaultSettings` and `config.metadata.defaultSettings` will prevail over `config.defaultSettings`, and a defined job setting will prevail over any `defaultSettings`

@xen-orchestra/backups/HealthCheckVmBackup.mjs

@@ -49,26 +49,25 @@ export class HealthCheckVmBackup {
         }
 
         // wait for the 'Running' event to be really stored in local xapi object cache
+
         restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
           timeout: remainingTimeout,
+          timeoutMessage: refOrUuid =>
+            `local xapi  did not get Running state for VM ${refOrUuid} after ${timeout / 1000} second`,
         })
 
         const running = new Date()
         remainingTimeout -= running - started
 
-        if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get Running state for VM ${restoredId} after ${timeout / 1000} second`)
-        }
-        // wait for the guest tool version to be defined
-        await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
+        // wait for the guest tools to be detected
+        await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_detected, {
           timeout: remainingTimeout,
+          timeoutMessage: refOrUuid =>
+            `timeout reached while waiting for ${refOrUuid} to report the driver version through the Xen tools. Please check or update the Xen tools.`,
         })
 
         const guestToolsReady = new Date()
         remainingTimeout -= guestToolsReady - running
-        if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get he guest tools check ${restoredId} after ${timeout / 1000} second`)
-        }
 
         if (waitForScript) {
           const startedRestoredVm = await xapi.waitObjectState(
@@ -79,19 +78,10 @@ export class HealthCheckVmBackup {
                 vm.xenstore_data['vm-data/xo-backup-health-check'] === 'failure'),
             {
               timeout: remainingTimeout,
+              timeoutMessage: refOrUuid =>
+                `timeout reached while waiting for ${refOrUuid} to report the startup script execution.`,
             }
           )
-          const scriptOk = new Date()
-          remainingTimeout -= scriptOk - guestToolsReady
-          if (remainingTimeout < 0) {
-            throw new Error(
-              `Backup health check script did not update vm-data/xo-backup-health-check of ${restoredId} after ${
-                timeout / 1000
-              } second, got ${
-                startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check']
-              } instead of 'success' or 'failure'`
-            )
-          }
 
           if (startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check'] !== 'success') {
             const message = startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check-error']

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -58,7 +58,7 @@ export class ImportVmBackup {
   async _reuseNearestSnapshot($defer, ignoredVdis) {
     const metadata = this._metadata
     const { mapVdisSrs } = this._importIncrementalVmSettings
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const streams = {}
     const metdataDir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
@@ -79,8 +79,13 @@ export class ImportVmBackup {
         debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
         for (const snapshotRef of xapiDisk.snapshots) {
           const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
-          debug('handling snapshot', { snapshot })
 
+          debug('handling snapshot', { snapshot })
+          if (snapshot.type === 'cbt_metadata') {
+            // disk without data can't be used as a base
+            debug('cbt metadata snapshot, skip')
+            continue
+          }
           // take only the first snapshot
           if (snapshotCandidate && snapshotCandidate.snapshot_time < snapshot.snapshot_time) {
             debug('already got a better candidate')
@@ -186,6 +191,7 @@ export class ImportVmBackup {
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 

@xen-orchestra/backups/README.md

@@ -12,6 +12,19 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backu
 npm install --save @xen-orchestra/backups
 ```
 
+## Usage
+
+## Changing the default backup configuration
+
+This library receives a backup configuration from `xo-server` or `xo-proxy`. This configuration is split between multiple properties:
+
+- `config` contains values related to the general backup configuration
+- `config.defaultSettings`, `config.vm.defaultSettings` and `config.metadata.defaultSettings` contain values related to backup jobs default settings:
+  - `config.vm.defaultSettings` is related to VM backup jobs (see [DEFAULT_XAPI_VM_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/VmsRemote.mjs#L15))
+  - `config.metadata.defaultSettings` is related to metadata backup jobs (see [DEFAULT_METADATA_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/Metadata.mjs#L12))
+  - `config.defaultSettings` is related to all backup jobs (see [DEFAULT_SETTINGS](https://github.com/vatesfr/xen-orchestra/blob/568e14f7eec7c3ddf685d324ad4733a3ef577995/%40xen-orchestra/backups/_runners/_Abstract.mjs#L7))
+  - in case of duplicate value, `config.vm.defaultSettings` and `config.metadata.defaultSettings` will prevail over `config.defaultSettings`, and a defined job setting will prevail over any `defaultSettings`
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -18,6 +18,7 @@ import fromEvent from 'promise-toolbox/fromEvent'
 import groupBy from 'lodash/groupBy.js'
 import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
+import reduce from 'lodash/reduce.js'
 import tar from 'tar'
 import zlib from 'zlib'
 
@@ -106,10 +107,13 @@ export class RemoteAdapter {
   async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
     yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
 
+    debug('activate LVM volume group', { vgName })
     await fromCallback(execFile, 'vgchange', ['-ay', vgName])
     try {
+      debug('get LVM volume group name and path', { vgName })
       yield lvs(['lv_name', 'lv_path'], vgName)
     } finally {
+      debug('deactivate LVM volume group', { vgName })
       await fromCallback(execFile, 'vgchange', ['-an', vgName])
     }
   }
@@ -120,15 +124,22 @@ export class RemoteAdapter {
       args.push('-o', partition.start * 512, '--sizelimit', partition.size)
     }
     args.push('--show', '-f', devicePath)
+
+    debug('attach loop device', { devicePath, partition })
     const path = (await fromCallback(execFile, 'losetup', args)).trim()
     try {
+      debug('list LVM physical volume', { path })
       await fromCallback(execFile, 'pvscan', ['--cache', path])
+
       yield path
     } finally {
       try {
         const vgNames = await pvs('vg_name', path)
+
+        debug('deactivate LVM volume groups', { vgNames })
         await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
       } finally {
+        debug('detach loop device', { path })
         await fromCallback(execFile, 'losetup', ['-d', path])
       }
     }
@@ -149,6 +160,7 @@ export class RemoteAdapter {
 
     const path = yield getTmpDir()
     const mount = options => {
+      debug('mount device', { devicePath, mountPath: path })
       return fromCallback(execFile, 'mount', [
         `--options=${options.join(',')}`,
         `--source=${devicePath}`,
@@ -166,6 +178,7 @@ export class RemoteAdapter {
     try {
       yield path
     } finally {
+      debug('umount device', { devicePath, mountPath: path })
       await fromCallback(execFile, 'umount', ['--lazy', path])
     }
   }
@@ -335,6 +348,8 @@ export class RemoteAdapter {
 
     const diskPath = handler.getFilePath('/' + diskId)
     const mountDir = yield getTmpDir()
+
+    debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
     await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
     try {
       let max = 0
@@ -356,6 +371,7 @@ export class RemoteAdapter {
 
       yield `${mountDir}/${maxEntry}`
     } finally {
+      debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
       await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
     }
   }
@@ -728,7 +744,7 @@ export class RemoteAdapter {
 
   async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
     const handler = this._handler
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const dir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
@@ -744,6 +760,7 @@ export class RemoteAdapter {
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 
@@ -826,6 +843,29 @@ export class RemoteAdapter {
     }
     return metadata
   }
+
+  #computeTotalBackupSizeRecursively(backups) {
+    return reduce(
+      backups,
+      (prev, backup) => {
+        const _backup = Array.isArray(backup) ? this.#computeTotalBackupSizeRecursively(backup) : backup
+        return {
+          onDisk: prev.onDisk + (_backup.onDisk ?? _backup.size),
+        }
+      },
+      { onDisk: 0 }
+    )
+  }
+
+  async getTotalVmBackupSize() {
+    return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups())
+  }
+
+  async getTotalBackupSize() {
+    const vmBackupSize = await this.getTotalVmBackupSize()
+    // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented by fs
+    return vmBackupSize
+  }
 }
 
 Object.assign(RemoteAdapter.prototype, {

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -1,4 +1,4 @@
-import test from 'test'
+import test from 'node:test'
 import { strict as assert } from 'node:assert'
 
 import tmp from 'tmp'

@xen-orchestra/backups/_cleanVm.mjs

@@ -121,7 +121,19 @@ export async function checkAliases(
 ) {
   const aliasFound = []
   for (const alias of aliasPaths) {
-    const target = await resolveVhdAlias(handler, alias)
+    let target
+    try {
+      target = await resolveVhdAlias(handler, alias)
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        logWarn('missing target of alias', { alias })
+        if (remove) {
+          logInfo('removing alias and non VHD target', { alias, target })
+          await handler.unlink(target)
+          await handler.unlink(alias)
+        }
+      }
+    }
 
     if (!isVhdFile(target)) {
       logWarn('alias references non VHD target', { alias, target })
@@ -201,9 +213,9 @@ export async function cleanVm(
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
-    if(removeTmp && basename(path)[0] === '.'){
+    if (removeTmp && basename(path)[0] === '.') {
       logInfo('deleting temporary VHD', { path })
-        return VhdAbstract.unlink(handler, path)
+      return VhdAbstract.unlink(handler, path)
     }
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {

@xen-orchestra/backups/_getOldEntries.mjs

@@ -1,4 +1,136 @@
-// returns all entries but the last retention-th
-export function getOldEntries(retention, entries) {
-  return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries
+import moment from 'moment-timezone'
+import assert from 'node:assert'
+
+function instantiateTimezonedDateCreator(timezone) {
+  return date => {
+    const transformed = timezone ? moment.tz(date, timezone) : moment(date)
+    assert.ok(transformed.isValid(), `date ${date} , timezone ${timezone} is invalid`)
+    return transformed
+  }
+}
+
+const LTR_DEFINITIONS = {
+  daily: {
+    makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.hour(copy.hour() - firstHourOfTheDay)
+        return copy.format('YYYY-MM-DD')
+      }
+    },
+  },
+  weekly: {
+    makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+
+        copy.date(date.date() - firstDayOfWeek)
+        // warning, the year in term of week may different from YYYY
+        // since the computation of the first week of a year is timezone dependant
+        return copy.format('gggg-WW')
+      }
+    },
+    ancestor: 'daily',
+  },
+  monthly: {
+    makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfMonth)
+        return copy.format('YYYY-MM')
+      }
+    },
+    ancestor: 'weekly',
+  },
+  yearly: {
+    makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfYear)
+        return copy.format('YYYY')
+      }
+    },
+    ancestor: 'monthly',
+  },
+}
+
+// returns all entries but the last retention-th
+/**
+ * return the entries too old to be kept
+ *  if multiple entries are i the same time bucket : keep only the most recent one
+ *  if an entry is valid in any of the bucket OR the  minRetentionCount : keep it
+ *  if a bucket is completly empty : it does not count as one, thus it may extend the retention
+ * @returns Array<Backup>
+ */
+export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
+  assert.strictEqual(
+    typeof minRetentionCount,
+    'number',
+    `minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  assert.strictEqual(
+    minRetentionCount >= 0,
+    true,
+    `minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  const dateBuckets = {}
+  const dateCreator = instantiateTimezonedDateCreator(timezone)
+  // only check buckets that have a retention set
+  for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
+    assert.notStrictEqual(LTR_DEFINITIONS[duration], undefined, `Retention of type ${duration} is not defined`)
+    assert.notStrictEqual(
+      timezone,
+      undefined,
+      `timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
+    )
+    assert.strictEqual(
+      typeof retention,
+      'number',
+      `retention  of type ${duration} must be a number, got ${JSON.stringify(retention)}`
+    )
+    assert.strictEqual(
+      retention > 0,
+      true,
+      `retention  of type ${duration} must be a positive number, got ${JSON.stringify(retention)}`
+    )
+    dateBuckets[duration] = {
+      remaining: retention,
+      lastMatchingBucket: null,
+      formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
+    }
+  }
+  const nb = entries.length
+  const oldEntries = []
+
+  for (let i = nb - 1; i >= 0; i--) {
+    const entry = entries[i]
+    const entryDate = dateCreator(entry.timestamp)
+    let shouldBeKept = false
+    for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
+      if (remaining === 0) {
+        continue
+      }
+      const bucket = formatter(entryDate)
+      if (lastMatchingBucket !== bucket) {
+        if (lastMatchingBucket !== null) {
+          assert.strictEqual(
+            lastMatchingBucket > bucket,
+            true,
+            `entries must be sorted in asc order ${lastMatchingBucket} ${bucket}`
+          )
+        }
+        shouldBeKept = true
+        dateBuckets[duration].remaining -= 1
+        dateBuckets[duration].lastMatchingBucket = bucket
+      }
+    }
+    if (i >= nb - minRetentionCount) {
+      shouldBeKept = true
+    }
+    if (!shouldBeKept) {
+      oldEntries.push(entry)
+    }
+  }
+  // we expect the entries to be in the right order
+  return oldEntries.reverse()
 }

@xen-orchestra/backups/_getOldEntries.test.mjs

@@ -0,0 +1,282 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { getOldEntries } from './_getOldEntries.mjs'
+
+describe('_getOldEntries() should succeed', () => {
+  const tests = [
+    {
+      args: [
+        1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+      ],
+      expectedIds: [1, 2],
+      testLabel: 'should  handle number based retention ',
+    },
+
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // too old
+          { timestamp: +new Date('2024-09-01 00:00:00'), id: 2 }, // too old
+          { timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // too old in same day
+          { timestamp: +new Date('2024-09-02 00:10:00'), id: 4 },
+          { timestamp: +new Date('2024-09-03 00:09:00'), id: 5 },
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // too old in same day
+          { timestamp: +new Date('2024-09-04 00:10:00'), id: 7 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 2, 3, 6],
+      testLabel: 'should  handle day based retention  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // week n-3 too old
+          { timestamp: +new Date('2024-09-02 00:00:00'), id: 2 }, // week n-3 too old
+          { timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 4 }, // week n-2
+          { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // week n-2
+          { timestamp: +new Date('2024-09-06 00:09:00'), id: 6 }, // week n-2
+          { timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2, most recent kept
+          { timestamp: +new Date('2024-09-09 00:09:00'), id: 8 }, // week n-1, too old
+          { timestamp: +new Date('2024-09-15 00:09:00'), id: 9 }, // week n-1  kept
+          { timestamp: +new Date('2024-09-22 00:09:00'), id: 10 }, // week n  kept
+        ],
+        {
+          longTermRetention: {
+            weekly: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 2, 3, 4, 5, 6, 8],
+      testLabel: 'should  handle week based retention  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-06-22 00:09:00'), id: 1 }, // too old
+          { timestamp: +new Date('2024-07-31 00:09:00'), id: 2 }, // first of july
+          { timestamp: +new Date('2024-08-01 00:09:00'), id: 3 }, // older of august
+          { timestamp: +new Date('2024-08-05 00:09:00'), id: 4 }, // older of august
+          { timestamp: +new Date('2024-08-07 00:09:00'), id: 5 }, // most recent of august
+          { timestamp: +new Date('2024-09-09 00:09:00'), id: 6 }, // older of september
+          { timestamp: +new Date('2024-09-15 00:09:00'), id: 7 }, // older of september
+          { timestamp: +new Date('2024-09-22 00:09:00'), id: 8 }, // most recent of september
+        ],
+        {
+          longTermRetention: {
+            monthly: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      expectedIds: [1, 3, 4, 6, 7],
+      testLabel: 'should  handle month based retention',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, // too old
+          { timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // too old in same year
+          { timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, // too old
+          { timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, // too old
+          { timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // too old
+          { timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, // new month,
+          { timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, // new month , week reached retention
+          { timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, // new week
+          { timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, // new week/month / year  daily reach retention
+          { timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, // same day/week/month/year
+          { timestamp: +new Date('2025-01-01 00:10:00'), id: 11 }, //  new  day / week / month
+          { timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // same day/week/month/year
+          { timestamp: +new Date('2025-12-31 00:09:00'), id: 13 }, // new month /year
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 2 },
+            weekly: { retention: 4 },
+            monthly: { retention: 5 },
+            yearly: { retention: 2 },
+          },
+          timezone: 'Europe/Paris', // use a time zone here because week definition is timezone bound
+        },
+      ],
+      expectedIds: [1, 2, 3, 4, 5, 10, 12],
+      testLabel: 'complete test  ',
+    },
+    {
+      args: [
+        0,
+        [
+          { timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // thrid day too old
+          { timestamp: +new Date('2024-09-01 00:10:00'), id: 2 }, // thrid day
+          { timestamp: +new Date('2024-09-01 00:20:00'), id: 3 }, // second day, too old
+          { timestamp: +new Date('2024-09-02 00:22:00'), id: 4 }, // second day too old
+          { timestamp: +new Date('2024-09-03 00:20:00'), id: 5 }, // second day in NZ
+          { timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // same day in NZ
+          { timestamp: +new Date('2024-09-04 00:10:00'), id: 7 }, // most recent
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 3 },
+          },
+          timezone: 'Pacific/Auckland', // GMT +6
+        },
+      ],
+      expectedIds: [1, 2, 3, 6],
+      testLabel: 'should  handle timezone ',
+    },
+  ]
+
+  for (const { args, expectedIds, testLabel } of tests) {
+    it(testLabel, () => {
+      const oldEntries = getOldEntries.apply(null, args)
+      assert.strictEqual(
+        oldEntries.length,
+        expectedIds.length,
+        `different length , ${JSON.stringify({ oldEntries, expectedIds })}`
+      )
+      for (let i = 0; i < expectedIds.length; i++) {
+        assert.strictEqual(
+          oldEntries[i].id,
+          expectedIds[i],
+          `different id , ${JSON.stringify({ i, expectedIds, oldEntries })}`
+        )
+      }
+    })
+  }
+})
+
+describe('_getOldEntries() should fail when called incorrectly', () => {
+  const tests = [
+    {
+      args: [
+        -1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+      ],
+      testLabel: 'negative min retention ',
+    },
+    {
+      args: [
+        { not: 'a number' },
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+      ],
+      testLabel: 'non number min retention ',
+    },
+    {
+      args: [
+        4,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 2, id: 2 },
+          { timestamp: 3, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            invalidType: { retention: 3 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'invalid type ltr ',
+    },
+    {
+      args: [
+        3,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 3, id: 2 },
+          { timestamp: 2, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 0 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'zero retention ltr ',
+    },
+    {
+      args: [
+        1,
+        [
+          { timestamp: 1, id: 1 },
+          { timestamp: 2, id: 2 },
+          { timestamp: 3, id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 'not a number' },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'no timezone ltr ',
+    },
+
+    {
+      args: [
+        5,
+        [
+          { timestamp: +new Date('2024-09-01'), id: 1 },
+          { timestamp: +new Date('2024-09-03'), id: 3 },
+          { timestamp: +new Date('2024-09-02'), id: 2 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 5 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'unsorted entries ',
+    },
+
+    {
+      args: [
+        5,
+        [
+          { timestamp: 'notadate', id: 1 },
+          { timestamp: 'still not a date, but after', id: 3 },
+        ],
+        {
+          longTermRetention: {
+            daily: { retention: 5 },
+          },
+          timezone: 'Europe/Paris',
+        },
+      ],
+      testLabel: 'broken date ',
+    },
+  ]
+
+  for (const { args, testLabel } of tests) {
+    it(testLabel, () => {
+      assert.throws(() => getOldEntries.apply(null, args), { code: 'ERR_ASSERTION' })
+    })
+  }
+})

@xen-orchestra/backups/_getTmpDir.mjs

@@ -1,16 +1,23 @@
 import Disposable from 'promise-toolbox/Disposable'
+import { createLogger } from '@xen-orchestra/log'
 import { join } from 'node:path'
 import { mkdir, rmdir } from 'node:fs/promises'
 import { tmpdir } from 'os'
 
+const { debug } = createLogger('xo:backups:getTmpDir')
+
 const MAX_ATTEMPTS = 3
 
 export async function getTmpDir() {
   for (let i = 0; true; ++i) {
     const path = join(tmpdir(), Math.random().toString(36).slice(2))
     try {
+      debug('creating directory', { path })
       await mkdir(path)
-      return new Disposable(() => rmdir(path), path)
+      return new Disposable(() => {
+        debug('removing directory', { path })
+        return rmdir(path)
+      }, path)
     } catch (error) {
       if (i === MAX_ATTEMPTS) {
         throw error

@xen-orchestra/backups/_incrementalVm.mjs

@@ -3,7 +3,6 @@ import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { CancelToken } from 'promise-toolbox'
 import { compareVersions } from 'compare-versions'
-import { createVhdStreamWithLength } from 'vhd-lib'
 import { defer } from 'golike-defer'
 
 import { cancelableMap } from './_cancelableMap.mjs'
@@ -53,14 +52,32 @@ export async function exportIncrementalVm(
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
     }
-
-    streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-      baseRef: baseVdi?.$ref,
-      cancelToken,
-      format: 'vhd',
-      nbdConcurrency,
-      preferNbd,
-    })
+    try {
+      streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
+        baseRef: baseVdi?.$ref,
+        cancelToken,
+        format: 'vhd',
+        nbdConcurrency,
+        preferNbd,
+      })
+    } catch (err) {
+      if (err.code === 'VDI_CANT_DO_DELTA') {
+        // fall back to a base
+        Task.info(`Can't do delta, will try to get a full stream`, { vdi })
+        streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
+          cancelToken,
+          format: 'vhd',
+          nbdConcurrency,
+          preferNbd,
+        })
+        // only warn if the fall back succeed
+        Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
+          vdi,
+        })
+      } else {
+        throw err
+      }
+    }
   })
 
   const suspendVdi = vm.$suspend_VDI
@@ -87,6 +104,18 @@ export async function exportIncrementalVm(
     }
   })
 
+  const vtpms = await Promise.all(
+    vm.$VTPMs.map(async vtpm => {
+      let content
+      try {
+        content = await vm.$xapi.call('VTPM.get_contents', vtpm.$ref)
+      } catch (err) {
+        console.error(err)
+      }
+      return content
+    })
+  )
+
   return Object.defineProperty(
     {
       version: '1.1.0',
@@ -96,6 +125,7 @@ export async function exportIncrementalVm(
       vm: {
         ...vm,
       },
+      vtpms,
     },
     'streams',
     {
@@ -227,9 +257,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
         if (typeof stream === 'function') {
           stream = await stream()
         }
-        if (stream.length === undefined) {
-          stream = await createVhdStreamWithLength(stream)
-        }
         await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
         await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
         await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
@@ -267,6 +294,12 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       }
     }),
   ])
+  // recreate VTPMs
+  await Promise.all(
+    (incrementalVm.vtpms ?? []).map(async contents => {
+      await xapi.VTPM_create({ VM: vmRef, contents })
+    })
+  )
 
   await Promise.all([
     incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),

@xen-orchestra/backups/_runners/_vmRunners/FullRemote.mjs

@@ -2,39 +2,39 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
-import { Task } from '../../Task.mjs'
 
 export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return FullRemoteWriter
   }
+
+  _filterTransferList(transferList) {
+    return transferList.filter(this._filterPredicate)
+  }
+
   async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
-    if (transferList.length > 0) {
-      for (const metadata of transferList) {
-        const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
-        const sizeContainer = watchStreamSize(stream)
+    for (const metadata of transferList) {
+      const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+      const sizeContainer = watchStreamSize(stream)
 
-        // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
-        await this._callWriters(
-          writer =>
-            writer.run({
-              stream: forkStreamUnpipe(stream),
-              // stream will be forked and transformed, it's not safe to attach additionnal properties to it
-              streamLength: stream.length,
-              timestamp: metadata.timestamp,
-              vm: metadata.vm,
-              vmSnapshot: metadata.vmSnapshot,
-              sizeContainer,
-            }),
-          'writer.run()'
-        )
-        // for healthcheck
-        this._tags = metadata.vm.tags
-      }
-    } else {
-      Task.info('No new data to upload for this VM')
+      // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
+      await this._callWriters(
+        writer =>
+          writer.run({
+            stream: forkStreamUnpipe(stream),
+            // stream will be forked and transformed, it's not safe to attach additionnal properties to it
+            streamLength: stream.length,
+            timestamp: metadata.timestamp,
+            vm: metadata.vm,
+            vmSnapshot: metadata.vmSnapshot,
+            sizeContainer,
+          }),
+        'writer.run()'
+      )
+      // for healthcheck
+      this._tags = metadata.vm.tags
     }
   }
 }

@xen-orchestra/backups/_runners/_vmRunners/IncrementalRemote.mjs

@@ -1,3 +1,5 @@
+import { createLogger } from '@xen-orchestra/log'
+
 import { asyncEach } from '@vates/async-each'
 import assert from 'node:assert'
 import * as UUID from 'uuid'
@@ -7,15 +9,25 @@ import mapValues from 'lodash/mapValues.js'
 import { AbstractRemote } from './_AbstractRemote.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
-import { Task } from '../../Task.mjs'
 import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
+const { warn } = createLogger('xo:backups:Incrementalremote')
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return IncrementalRemoteWriter
   }
+
+  // we'll transfer the full list if at least one backup should be transfered
+  // to ensure we don't cut the delta chain
+  _filterTransferList(transferList) {
+    if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
+      return transferList
+    }
+    return []
+  }
+
   async _selectBaseVm(metadata) {
     // for each disk , get the parent
     const baseUuidToSrcVdi = new Map()
@@ -37,11 +49,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
     })
 
     const presentBaseVdis = new Map(baseUuidToSrcVdi)
-    await this._callWriters(
-      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
-      'writer.checkBaseVdis()',
-      false
-    )
+    await this._callWriters(writer => writer.checkBaseVdis(presentBaseVdis), 'writer.checkBaseVdis()', false)
     // check if the parent vdi are present in all the remotes
     baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
       if (!presentBaseVdis.has(baseUuid)) {
@@ -53,50 +61,58 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
 
-    if (transferList.length > 0) {
-      for (const metadata of transferList) {
-        assert.strictEqual(metadata.mode, 'delta')
-        await this._selectBaseVm(metadata)
-        await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
-        const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
-          useChain: false,
+    for (const metadata of transferList) {
+      assert.strictEqual(metadata.mode, 'delta')
+      const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
+        useChain: false,
+      })
+      // don't trust metadata too much
+      // recompute if it's a base backup
+      // recompute if disks are differencing or not
+      const isVhdDifferencing = {}
+
+      await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
+        isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
+      })
+      const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
+      if (metadata.isBase === hasDifferencingDisk) {
+        warn(`Metadata isBase and real disk value are different`, {
+          metadataIsBase: metadata.isBase,
+          diskIsBase: !hasDifferencingDisk,
+          isVhdDifferencing,
         })
-
-        const isVhdDifferencing = {}
-
-        await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-          isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
-        })
-
-        incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
-        await this._callWriters(
-          writer =>
-            writer.transfer({
-              deltaExport: forkDeltaExport(incrementalExport),
-              isVhdDifferencing,
-              timestamp: metadata.timestamp,
-              vm: metadata.vm,
-              vmSnapshot: metadata.vmSnapshot,
-            }),
-          'writer.transfer()'
-        )
-        // this will update parent name with the needed alias
-        await this._callWriters(
-          writer =>
-            writer.updateUuidAndChain({
-              isVhdDifferencing,
-              timestamp: metadata.timestamp,
-              vdis: incrementalExport.vdis,
-            }),
-          'writer.updateUuidAndChain()'
-        )
-
-        await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
-        // for healthcheck
-        this._tags = metadata.vm.tags
       }
-    } else {
-      Task.info('No new data to upload for this VM')
+      metadata.isBase = !hasDifferencingDisk
+      metadata.isVhdDifferencing = isVhdDifferencing
+      await this._selectBaseVm(metadata)
+      await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
+
+      incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
+      await this._callWriters(
+        writer =>
+          writer.transfer({
+            deltaExport: forkDeltaExport(incrementalExport),
+            isVhdDifferencing,
+            timestamp: metadata.timestamp,
+            vm: metadata.vm,
+            vmSnapshot: metadata.vmSnapshot,
+          }),
+        'writer.transfer()'
+      )
+      // this will update parent name with the needed alias
+      await this._callWriters(
+        writer =>
+          writer.updateUuidAndChain({
+            isVhdDifferencing,
+            timestamp: metadata.timestamp,
+            vdis: incrementalExport.vdis,
+          }),
+        'writer.updateUuidAndChain()'
+      )
+
+      await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
+      // for healthcheck
+      this._tags = metadata.vm.tags
     }
   }
 }

@xen-orchestra/backups/_runners/_vmRunners/IncrementalXapi.mjs

@@ -91,9 +91,11 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       'writer.updateUuidAndChain()'
     )
 
-    if (!isFull) {
+    if (isFull) {
+      await setVmDeltaChainLength(this._xapi, exportedVm.$ref, 0)
+    } else {
       await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
-    } // on a full the delta chain will be null
+    }
 
     // not the case if offlineBackup
     if (exportedVm.is_a_snapshot) {

@xen-orchestra/backups/_runners/_vmRunners/_Abstract.mjs

@@ -74,9 +74,12 @@ export const Abstract = class AbstractVmBackupRunner {
 
     // check if current VM has tags
     const tags = this._tags
-    const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
 
-    if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
+    // accept both 'xo:no-health-check' and 'xo:no-health-check=reason'
+    const vmAlwaysIgnored = tags.some(t => t.startsWith('xo:no-health-check'))
+    const intersect = !vmAlwaysIgnored && settings.healthCheckVmsWithTags.some(t => tags.includes(t))
+
+    if (vmAlwaysIgnored || (settings.healthCheckVmsWithTags.length !== 0 && !intersect)) {
       // create a task to have an info in the logs and reports
       return Task.run(
         {

@xen-orchestra/backups/_runners/_vmRunners/_AbstractRemote.mjs

@@ -1,14 +1,18 @@
-import { asyncEach } from '@vates/async-each'
+import groupBy from 'lodash/groupBy.js'
+
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
 import { Disposable } from 'promise-toolbox'
+import { createPredicate } from 'value-matcher'
 
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
 import { Abstract } from './_Abstract.mjs'
 import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
+import { Task } from '../../Task.mjs'
 
 export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
+  _filterPredicate
   constructor({
     config,
     job,
@@ -57,39 +61,74 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
         })
       )
     })
+    const { filter } = job
+    if (filter === undefined) {
+      this._filterPredicate = () => true
+    } else {
+      this._filterPredicate = createPredicate(filter)
+    }
   }
 
-  async _computeTransferList(predicate) {
-    const vmBackups = await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, predicate)
+  async #computeTransferListPerJob(sourceBackups, remotesBackups) {
     const localMetada = new Map()
-    Object.values(vmBackups).forEach(metadata => {
+    sourceBackups.forEach(metadata => {
       const timestamp = metadata.timestamp
       localMetada.set(timestamp, metadata)
     })
-    const nbRemotes = Object.keys(this.remoteAdapters).length
+    const nbRemotes = remotesBackups.length
     const remoteMetadatas = {}
-    await asyncEach(Object.values(this.remoteAdapters), async remoteAdapter => {
-      const remoteMetadata = await remoteAdapter.listVmBackups(this._vmUuid, predicate)
-      remoteMetadata.forEach(metadata => {
+    remotesBackups.forEach(async remoteBackups => {
+      remoteBackups.forEach(metadata => {
         const timestamp = metadata.timestamp
         remoteMetadatas[timestamp] = (remoteMetadatas[timestamp] ?? 0) + 1
       })
     })
 
-    let chain = []
+    let transferList = []
     const timestamps = [...localMetada.keys()]
     timestamps.sort()
     for (const timestamp of timestamps) {
       if (remoteMetadatas[timestamp] !== nbRemotes) {
         // this backup is not present in all the remote
         // should be retransfered if not found later
-        chain.push(localMetada.get(timestamp))
+        transferList.push(localMetada.get(timestamp))
       } else {
         // backup is present in local and remote : the chain has already been transferred
-        chain = []
+        transferList = []
       }
     }
-    return chain
+    if (transferList.length > 0) {
+      const filteredTransferList = this._filterTransferList(transferList)
+      if (filteredTransferList.length > 0) {
+        return filteredTransferList
+      } else {
+        Task.info('This VM is excluded by the job filter')
+        return []
+      }
+    } else {
+      Task.info('No new data to upload for this VM')
+    }
+
+    return []
+  }
+
+  /**
+   *
+   * @param {*} vmPredicate a callback checking if backup is eligible for transfer. This filter MUST NOT cut delta chains
+   * @returns
+   */
+  async _computeTransferList(vmPredicate) {
+    const sourceBackups = Object.values(await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
+    const remotesBackups = await Promise.all(
+      Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
+    )
+    const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
+    const transferByJobs = await Promise.all(
+      Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
+        this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
+      )
+    )
+    return transferByJobs.flat(1)
   }
 
   async run($defer) {

@xen-orchestra/backups/_runners/_vmRunners/_AbstractXapi.mjs

@@ -150,7 +150,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           try {
             // enable CBT on all disks if possible
             const diskRefs = await xapi.VM_getDisks(vm.$ref)
-            await Promise.all(diskRefs.map(diskRef => xapi.call('VDI.enable_cbt', diskRef)))
+            await Promise.all(diskRefs.map(diskRef => xapi.callAsync('VDI.enable_cbt', diskRef)))
           } catch (error) {
             Task.info(`couldn't enable CBT`, error)
           }
@@ -203,7 +203,13 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     for (const srcVdi of srcVdis) {
       const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
       for (const snapshot of snapshots) {
-        if (snapshot.other_config[JOB_ID] === jobId) {
+        // only keep the snapshot related to this backup job
+        // and only if the job is still using  purge snapshot data or if the disk
+        // is not a cbt metadata disk ( expect a type: user for normal disks)
+        if (
+          snapshot.other_config[JOB_ID] === jobId &&
+          (this._settings.cbtDestroySnapshotData || snapshot.type !== 'cbt_metadata')
+        ) {
           this._jobSnapshotVdis.push(snapshot)
         }
       }
@@ -214,8 +220,20 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     const allSettings = this.job.settings
     const baseSettings = this._baseSettings
 
-    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     const xapi = this._xapi
+    // ensure all the event has been processed by the xapi
+    await xapi.barrier()
+    // ensure cached object are up to date
+    this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
+
+    // get the datetime of the most recent snapshot
+    const lastSnapshotDateTime = this._jobSnapshotVdis
+      .map(({ other_config }) => other_config[DATETIME])
+      .sort()
+      .pop()
+
+    // remove older snapshot schedule per schedule
+    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
       const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
 
@@ -227,21 +245,23 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         ...allSettings[scheduleId],
         ...allSettings[this._vm.uuid],
       }
-      // ensure we never delete the last one for delta
-      const minRetention = this.job.mode === 'delta' ? 1 : 0
-      const retention = Math.max(settings.snapshotRetention ?? 0, minRetention)
+      const retention = settings.snapshotRetention ?? 0
       await asyncMap(getOldEntries(retention, datetimes), async datetime => {
+        // keep the last snapshot across all schedules for delta
+        // since we'll need it to compute delta for next backup
+        if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
+          return
+        }
         const vdis = snapshotPerDatetime[datetime]
         let vmRef
         // if there is an attached VM => destroy the VM (Non CBT backups)
         for (const vdi of vdis) {
-          if (vdi.$VBDs.length > 0) {
-            const vbds = vdi.$VBDs
+          const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
+          if (vbds.length > 0) {
             // only one VM linked to this vdi
             // this will throw error for VDI still attached to control domain
             assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
             const vm = vbds[0].$VM
-            assert.strictEqual(vm.is_control_domain, false, `Disk is still attached to DOM0 VM`) // don't delete a VM (especially a control domain)
             assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
 
             const vmRefVdi = vm.$ref
@@ -287,10 +307,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       for (const vdiRef of vdiRefs) {
         try {
           // data_destroy will fail with a VDI_NO_CBT_METADATA error if CBT is not enabled on this VDI
-          await this._xapi.call('VDI.data_destroy', vdiRef)
+          await this._xapi.VDI_dataDestroy(vdiRef)
           Task.info(`Snapshot data has been deleted`, { vdiRef })
         } catch (error) {
-          Task.warning(`Couldn't deleted snapshot data`, { error, vdiRef })
+          Task.warning(`Couldn't delete snapshot data`, { error, vdiRef })
         }
       }
     }
@@ -335,15 +355,22 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       const reason = 'VM migration is blocked during backup'
       await vm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
 
-      $defer(() =>
+      $defer(async () => {
         // delete the entries if they did not exist previously or if they were
         // equal to reason (which happen if a previous backup was interrupted
         // before resetting them)
-        vm.update_blocked_operations({
+        await vm.update_blocked_operations({
           migrate_send: migrate_send === undefined || migrate_send === reason ? null : migrate_send,
           pool_migrate: pool_migrate === undefined || pool_migrate === reason ? null : pool_migrate,
         })
-      )
+
+        // 2024-08-19 - Work-around a XAPI bug where allowed_operations are not properly computed when blocked_operations is updated
+        //
+        // this is a problem because some clients (e.g. XenCenter) use this field to allow operations.
+        //
+        // internal source: https://team.vates.fr/vates/pl/mjmxnce9qfdx587r3qpe4z91ho
+        await vm.$call('update_allowed_operations')
+      })
     }
 
     await this._fetchJobSnapshots()

@xen-orchestra/backups/_runners/_writers/FullRemoteWriter.mjs

@@ -38,7 +38,8 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
 
     const oldBackups = getOldEntries(
       settings.exportRetention - 1,
-      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'full' && _.scheduleId === scheduleId)
+      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'full' && _.scheduleId === scheduleId),
+      { longTermRetention: settings.longTermRetention, timezone: settings.timezone }
     )
     const deleteOldBackups = () => adapter.deleteFullVmBackups(oldBackups)
 

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -66,7 +66,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
   async beforeBackup() {
     await super.beforeBackup()
-    return this._cleanVm({ merge: true })
+    return this._cleanVm({ merge: true, remove: true })
   }
 
   prepare({ isFull }) {
@@ -95,7 +95,8 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
     const oldEntries = getOldEntries(
       settings.exportRetention - 1,
-      await adapter.listVmBackups(vmUuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId)
+      await adapter.listVmBackups(vmUuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId),
+      { longTermRetention: settings.longTermRetention, timezone: settings.timezone }
     )
     this._oldEntries = oldEntries
 
@@ -149,6 +150,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
         assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
         // forbid any kind of loop
         assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
+        // re-chainVhd is mandatory
+        // since the parent may be a alias or not
+        // and the child may be the other
         await chainVhd(handler, parentPath, handler, path)
       }
 
@@ -217,6 +221,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       vhds,
       vm,
       vmSnapshot,
+      vtpms: deltaExport.vtpms,
     }
 
     const { size } = await Task.run({ name: 'transfer' }, async () => {

@xen-orchestra/backups/_runners/_writers/IncrementalXapiWriter.mjs

@@ -15,11 +15,15 @@ import assert from 'node:assert'
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
     const sr = this._sr
+    if (baseUuidToSrcVdi.size === 0) {
+      // searching for the vdis is expensive
+      // don't do it if there is nothing to find
+      return
+    }
 
     // @todo use an index if possible
     // @todo : this seems similare to decorateVmMetadata
-
-    const replicatedVdis = sr.$VDIs 
+    const replicatedVdis = sr.$VDIs
       .filter(vdi => {
         // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
         return baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
@@ -103,11 +107,10 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       .filter(_ => !!_)
     // @todo use index ?
 
-    const replicatedVdis = sr.$VDIs
-      .filter(vdi => {
-        // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-        return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
-      })
+    const replicatedVdis = sr.$VDIs.filter(vdi => {
+      // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
+      return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
+    })
 
     Object.values(backup.vdis).forEach(vdi => {
       vdi.other_config[COPY_OF] = vdi.uuid

@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs

@@ -58,8 +58,8 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
       const { disableMergeWorker } = this._config
       // merge worker only compatible with local remotes
       const { handler } = this._adapter
-      const willMergeInWorker = !disableMergeWorker && typeof handler.getRealPath === 'function'
-
+      const synchronousMerge = this._settings.mergeBackupsSynchronously
+      const willMergeInWorker = !synchronousMerge && !disableMergeWorker && typeof handler.getRealPath === 'function'
       const { merge } = await this._cleanVm({ remove: true, merge: !willMergeInWorker })
       await this.#lock.dispose()
 

@xen-orchestra/backups/docs/VM backups/README.md

@@ -65,7 +65,7 @@ In case any incoherence is detected, the file is deleted so it will be fully gen
   - `xo:backup:deltaChainLength` = n (number of delta copies/replicated since a full)
   - `xo:backup:exported` = 'true' (added at the end of the backup)
 
-### Of created VMs , their associated VDIs and snapshots 
+### Of created VMs , their associated VDIs and snapshots
 
 - `other_config`:
   - `xo:backup:datetime`: format is UTC %Y%m%dT%H:%M:%SZ

@xen-orchestra/backups/package.json

@@ -8,13 +8,13 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.52.2",
+  "version": "0.57.0",
   "engines": {
     "node": ">=14.18"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "node--test *.integ.mjs"
+    "test-integration": "node --test *.integ.mjs"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
@@ -23,13 +23,13 @@
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.1.0",
-    "@vates/disposable": "^0.1.5",
-    "@vates/fuse-vhd": "^2.1.1",
-    "@vates/nbd-client": "^3.1.0",
+    "@vates/disposable": "^0.1.6",
+    "@vates/fuse-vhd": "^2.1.2",
+    "@vates/nbd-client": "^3.1.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.1.7",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/fs": "^4.3.0",
+    "@xen-orchestra/log": "^0.7.1",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
     "compare-versions": "^6.0.0",
@@ -39,6 +39,7 @@
     "human-format": "^1.2.0",
     "limit-concurrency-decorator": "^0.6.0",
     "lodash": "^4.17.20",
+    "moment-timezone": "^0.5.46",
     "ms": "^2.1.3",
     "node-zone": "^0.4.0",
     "parse-pairs": "^2.0.0",
@@ -46,19 +47,19 @@
     "proper-lockfile": "^4.1.2",
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.11.0",
-    "xen-api": "^4.2.0",
+    "value-matcher": "^0.2.0",
+    "vhd-lib": "^4.11.2",
+    "xen-api": "^4.7.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
     "fs-extra": "^11.1.0",
     "rimraf": "^5.0.1",
     "sinon": "^18.0.0",
-    "test": "^3.2.1",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^7.3.0"
+    "@xen-orchestra/xapi": "^7.10.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^4.2.0"
+    "xen-api": "^4.7.0"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/cron/index.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const test = require('test')
+const test = require('node:test')
 const assert = require('assert').strict
 const sinon = require('sinon')
 

@xen-orchestra/cron/next.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('assert').strict
 
 const mapValues = require('lodash/mapValues')

@xen-orchestra/cron/package.json

@@ -39,10 +39,9 @@
   },
   "scripts": {
     "postversion": "npm publish",
-    "test": "node--test"
+    "test": "node --test"
   },
   "devDependencies": {
-    "sinon": "^18.0.0",
-    "test": "^3.2.1"
+    "sinon": "^18.0.0"
   }
 }

@xen-orchestra/cron/parse.test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { describe, it } = require('test')
+const { describe, it } = require('node:test')
 const assert = require('assert').strict
 
 const parse = require('./parse')

@xen-orchestra/defined/.USAGE.md

@@ -0,0 +1,29 @@
+### `defined()`
+
+Returns the first non-`undefined` value from a list of arguments, evaluating functions if needed.
+
+There is two ways to use this function:
+
+- either with a single array argument: it should return the first non-undefined item
+- or with multiple arguments: it should return the first non-undefined argument
+
+If only `undefined` values, return `undefined`.
+
+```js
+import defined from '@xen-orchestra/defined/index.js'
+
+defined(undefined, 'foo', 42)
+// Returns 'foo'
+
+defined([undefined, null, 10])
+// Returns null
+
+defined([undefined, undefined], [undefined, undefined, 10])
+// Returns [undefined, undefined]
+
+defined(() => 'bar', 42)
+// Returns 'bar'
+
+defined(undefined, undefined)
+// Returns undefined
+```

@xen-orchestra/defined/README.md

@@ -14,6 +14,38 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/defin
 npm install --save @xen-orchestra/defined
 ```
 
+## Usage
+
+### `defined()`
+
+Returns the first non-`undefined` value from a list of arguments, evaluating functions if needed.
+
+There is two ways to use this function:
+
+- either with a single array argument: it should return the first non-undefined item
+- or with multiple arguments: it should return the first non-undefined argument
+
+If only `undefined` values, return `undefined`.
+
+```js
+import defined from '@xen-orchestra/defined/index.js'
+
+defined(undefined, 'foo', 42)
+// Returns 'foo'
+
+defined([undefined, null, 10])
+// Returns null
+
+defined([undefined, undefined], [undefined, undefined, 10])
+// Returns [undefined, undefined]
+
+defined(() => 'bar', 42)
+// Returns 'bar'
+
+defined(undefined, undefined)
+// Returns undefined
+```
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@xen-orchestra/defined/index.js

@@ -22,7 +22,7 @@ function defined() {
   }
 
   for (let i = 0; i < n; ++i) {
-    let arg = arguments[i]
+    let arg = args[i]
     if (typeof arg === 'function') {
       arg = get(arg)
     }

@xen-orchestra/defined/index.test.mjs

@@ -0,0 +1,30 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import defined_ from '@xen-orchestra/defined/index.js'
+
+// for each test, use the flat syntax and the array syntax
+for (const [title, defined] of Object.entries({
+  'defined(...values)': defined_,
+  'defined([ ...values ])': (...args) => defined_(args),
+})) {
+  describe(title, () => {
+    it('returns the first non undefined value', () => {
+      assert.deepEqual(defined(undefined, 'foo', 42), 'foo')
+    })
+
+    it('returns undefined if only undefined values', () => {
+      assert.equal(defined(undefined, undefined), undefined)
+    })
+
+    it('resolves functions with `get(fn)`', () => {
+      const o = { foo: undefined, baz: 'baz' }
+      assert.deepEqual(
+        defined(
+          () => o.foo.bar,
+          () => o.baz
+        ),
+        'baz'
+      )
+    })
+  })
+}