fix: add debug to better understand timeout error

The docs on devops tools and kubernetes was moved
from the Vates VMS doc to the Xen Orchestra doc.
As a result, one link to the kubernetes page broke in the XO doc.
This commit fixes that broken link.

Signed-off-by: Thomas Moraine <thomas.moraine@vates.tech>

.eslintrc.js

@@ -281,7 +281,7 @@ module.exports = {
       },
     },
     {
-      files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
+      files: ['@xen-orchestra/{web-core,lite,web}/src/route-map.d.ts'],
       parserOptions: {
         sourceType: 'module',
       },

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Setup Node environment
         uses: actions/setup-node@v4
         with:
-          node-version: '24.14'
+          node-version: '>=22.12'
           cache: 'yarn'
 
       - name: Install project dependencies

.prettierignore

@@ -3,4 +3,5 @@
 **/*.hbs
 
 # Generated files
-/@xen-orchestra/web/typed-router.d.ts
+/@xen-orchestra/web/src/route-map.d.ts
+/@xen-orchestra/lite/src/route-map.d.ts

@vates/fuse-vhd/index.mjs

@@ -1,12 +1,12 @@
-import LRU from 'lru-cache'
 import Fuse from 'fuse-native'
+import LRU from 'lru-cache'
 import { createLogger } from '@xen-orchestra/log'
 import { VhdSynthetic } from 'vhd-lib'
 import { Disposable, fromCallback } from 'promise-toolbox'
 
 const { warn } = createLogger('vates:fuse-vhd')
 
-// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
+// build a stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
   mtime: st.mtime || new Date(),
   atime: st.atime || new Date(),
@@ -20,42 +20,109 @@ const stat = st => ({
 export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
   const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
 
-  const cache = new LRU({
-    max: 16, // each cached block is 2MB in size
-  })
   await vhd.readBlockAllocationTable()
+  const { blockSize } = vhd.header
+
+  // A zeroed buffer returned for VHD blocks that are not present in the chain.
+  // Never mutated — safe to share across all reads.
+  const EMPTY_BLOCK = Buffer.alloc(blockSize, 0)
+
+  // --- coalescing block read queue ---
+  //
+  // FUSE read requests from NTFS-3g arrive in small chunks (typically 4 KiB)
+  // while VHD blocks are 2 MiB. Reading blocks one at a time from the backing
+  // store (CIFS/NFS/…) bounds libuv threadpool usage to 1 concurrent I/O call.
+  //
+  // Coalescing: all pending requests for the same blockId share a single
+  // readBlock() call. When the block resolves, every waiter is fulfilled at once.
+  //
+  // Together these two properties prevent the FUSE-on-FUSE threadpool deadlock:
+  // concurrent NTFS-3g reads no longer starve the VHD block reads they depend on.
+  // In-memory block cache. Each VHD block is 2 MiB; 16 blocks = 32 MiB.
+  // This is the primary tool against the 300× CIFS amplification: NTFS-3g
+  // reads a 2 MiB block in ~512 sequential 4 KiB FUSE requests. Without a
+  // cache, each request causes a full 2 MiB CIFS read. With the cache, only
+  // the first request reads from CIFS; the rest are served from RAM.
+  const blockCache = new LRU({ max: 64 })
+
+  const pendingBlocks = new Map() // blockId → Promise<Buffer>
+  const readQueue = [] // { blockId, resolve, reject }[]
+  let queueRunning = false
+
+  // log stats every 10 s while there is activity
+
+  function enqueueBlock(blockId) {
+    // 1. serve from cache if available
+    if (blockCache.has(blockId)) {
+      return Promise.resolve(blockCache.get(blockId))
+    }
+    // 2. coalesce: join an existing in-flight read for this block
+    if (pendingBlocks.has(blockId)) {
+      return pendingBlocks.get(blockId)
+    }
+    const p = new Promise((resolve, reject) => {
+      readQueue.push({ blockId, resolve, reject })
+    })
+    pendingBlocks.set(blockId, p)
+    p.then(
+      data => {
+        pendingBlocks.delete(blockId)
+        blockCache.set(blockId, data)
+      },
+      () => pendingBlocks.delete(blockId)
+    )
+    drainQueue()
+    return p
+  }
+
+  function drainQueue() {
+    if (queueRunning) return
+    queueRunning = true
+    // async IIFE — errors are forwarded to individual reject() callbacks,
+    // so the outer floating promise is always fulfilled
+    ;(async () => {
+      while (readQueue.length > 0) {
+        const { blockId, resolve, reject } = readQueue.shift()
+        try {
+          resolve((await vhd.readBlock(blockId)).data)
+        } catch (err) {
+          reject(err)
+        }
+      }
+      queueRunning = false
+    })()
+  }
+
+  async function readData(buf, len, pos) {
+    if (len === 0) return 0
+    const startBlockId = Math.floor(pos / blockSize)
+    // use (pos + len - 1) so an exact block boundary doesn't include the next block
+    const endBlockId = Math.floor((pos + len - 1) / blockSize)
+
+    let copied = 0
+    for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
+      const data = vhd.containsBlock(blockId) ? await enqueueBlock(blockId) : EMPTY_BLOCK
+      const offsetStart = blockId === startBlockId ? pos % blockSize : 0
+      const offsetEnd = blockId === endBlockId ? ((pos + len - 1) % blockSize) + 1 : blockSize
+      data.copy(buf, copied, offsetStart, offsetEnd)
+      copied += offsetEnd - offsetStart
+    }
+    return copied
+  }
+
   const fuse = new Fuse(mountDir, {
     async readdir(path, cb) {
-      if (path === '/') {
-        return cb(null, ['vhd0'])
-      }
+      if (path === '/') return cb(null, ['vhd0'])
       cb(Fuse.ENOENT)
     },
     async getattr(path, cb) {
-      if (path === '/') {
-        return cb(
-          null,
-          stat({
-            mode: 'dir',
-            size: 4096,
-          })
-        )
-      }
-      if (path === '/vhd0') {
-        return cb(
-          null,
-          stat({
-            mode: 'file',
-            size: vhd.footer.currentSize,
-          })
-        )
-      }
-
+      if (path === '/') return cb(null, stat({ mode: 'dir', size: 4096 }))
+      if (path === '/vhd0') return cb(null, stat({ mode: 'file', size: vhd.footer.currentSize }))
       cb(Fuse.ENOENT)
     },
     read(path, fd, buf, len, pos, cb) {
       if (path === '/vhd0') {
-        return vhd.readRawData(pos, len, cache, buf).then(cb, error => {
+        return readData(buf, len, pos).then(cb, error => {
           warn('read error', { path, len, pos, error })
           cb(Fuse.EIO)
         })
@@ -64,7 +131,9 @@ export const mount = Disposable.factory(async function* mount(handler, diskPath,
     },
   })
   return new Disposable(
-    () => fromCallback(cb => fuse.unmount(cb)),
+    () => {
+      return fromCallback(cb => fuse.unmount(cb))
+    },
     fromCallback(cb => fuse.mount(cb))
   )
 })

@vates/fuse-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/fuse-vhd",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "license": "ISC",
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -19,12 +19,12 @@
   },
   "main": "./index.mjs",
   "dependencies": {
-    "@xen-orchestra/fs": "^4.8.0",
+    "@xen-orchestra/fs": "^4.9.3",
     "@xen-orchestra/log": "^0.7.2",
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.16.0"
+    "vhd-lib": "^4.16.1"
   },
   "bin": {
     "xo-fuse-vhd": "./cli.mjs"

@vates/generator-toolbox/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/generator-toolbox",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "main": "dist/index.mjs",
   "license": "MIT",
   "private": false,
@@ -10,7 +10,7 @@
     "rimraf": "^6.0.1",
     "@types/node": "^20.6",
     "typescript": "~5.6",
-    "typescript-eslint": "^8.23.0"
+    "typescript-eslint": "^8.61.0"
   },
   "scripts": {
     "build": "tsc",

@vates/http-server-plus/README.md

@@ -2,8 +2,18 @@
 
 # @vates/http-server-plus
 
+[![Package Version](https://badgen.net/npm/v/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus) ![License](https://badgen.net/npm/license/@vates/http-server-plus) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/http-server-plus)](https://bundlephobia.com/result?p=@vates/http-server-plus) [![Node compatibility](https://badgen.net/npm/node/@vates/http-server-plus)](https://npmjs.org/package/@vates/http-server-plus)
+
 > Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance
 
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/http-server-plus):
+
+```sh
+npm install --save @vates/http-server-plus
+```
+
 ## Usage
 
 ### `create([opts], [requestListener])`
@@ -48,10 +58,7 @@ await server.listen({ systemdSocket: 0 })
 Multiple endpoints can be started on the same instance:
 
 ```js
-const [httpAddr, httpsAddr] = await Promise.all([
-  server.listen({ port: 80 }),
-  server.listen({ port: 443, cert, key }),
-])
+const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
 console.log('listening on', httpAddr, httpsAddr)
 ```
 

@vates/http-server-plus/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/http-server-plus",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "license": "ISC",
   "description": "Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance",
   "keywords": [
@@ -27,9 +27,10 @@
   },
   "scripts": {
     "pretest": "eslint .",
-    "test": "node --test index.spec.js"
+    "test": "node --test index.spec.js",
+    "postversion": "npm publish --access public"
   },
-  "private": true,
+  "private": false,
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"

@vates/nbd-client/NbdDisk.mjs

@@ -19,6 +19,12 @@ export class NbdDisk extends RandomAccessDisk {
   /** @type {number} */
   #blockSize
 
+  /** @type {number | undefined} */
+  #hasBlockCursor
+
+  /** @type {number | undefined} */
+  #hasBlockPreviousIndex
+
   constructor(nbdInfos, blockSize, { dataMap } = {}) {
     super()
     this.#blockSize = blockSize
@@ -27,11 +33,38 @@ export class NbdDisk extends RandomAccessDisk {
   }
 
   #processDatamap(rawDataMap) {
-    return rawDataMap
-      .filter(({ type }) => type === 0)
+    const ranges = rawDataMap
+      .filter(({ type, length }) => type === 0 && length > 0)
       .map(({ offset, length }) => ({ offset, length }))
       .sort(({ offset: offset1 }, { offset: offset2 }) => offset1 - offset2)
+
+    // hasBlock()'s forward-only cursor is only correct if the extents are
+    // sorted AND disjoint: an earlier extent must never reach into a block
+    // after a later one, otherwise the cursor could skip it and wrongly report
+    // a block as empty, dropping data and corrupting the output.
+    // Touching extents (cur.offset === prevEnd) are merged
+    const merged = []
+    for (const range of ranges) {
+      const last = merged[merged.length - 1]
+      if (last !== undefined) {
+        const lastEnd = last.offset + last.length
+        if (range.offset < lastEnd) {
+          throw new Error(
+            `overlapping ranges in data map: [${last.offset}, ${lastEnd}) and [${range.offset}, ${range.offset + range.length})`
+          )
+        }
+        if (range.offset === lastEnd) {
+          // touching: extend the previous extent
+          last.length += range.length
+          continue
+        }
+      }
+      // gap (or first range): new extent
+      merged.push(range)
+    }
+    return merged
   }
+
   /**
    * @param {number} index
    * @returns {Promise<DiskBlock>}
@@ -117,6 +150,34 @@ export class NbdDisk extends RandomAccessDisk {
     return [...indexes].sort((a, b) => a - b)
   }
 
+  /**
+   * Counts the allocated blocks without materializing the index list.
+   *
+   * @returns {number}
+   */
+  getBlockIndexesCount() {
+    if (!this.#dataMap) {
+      throw new Error("can't getBlockIndexesCount before init")
+    }
+
+    const blockSize = this.getBlockSize()
+
+    let count = 0
+    let lastCountedBlock = -1
+    for (const { offset, length } of this.#dataMap) {
+      const firstBlockIndex = Math.floor(offset / blockSize)
+      const lastBlockIndex = Math.floor((offset + length - 1) / blockSize)
+      const from = Math.max(firstBlockIndex, lastCountedBlock + 1)
+
+      if (lastBlockIndex >= from) {
+        count += lastBlockIndex - from + 1
+        lastCountedBlock = lastBlockIndex
+      }
+    }
+
+    return count
+  }
+
   /**
    * @param {number} index
    * @returns {boolean}
@@ -125,8 +186,32 @@ export class NbdDisk extends RandomAccessDisk {
     if (!this.#dataMap) {
       throw new Error("can't hasBlock before init")
     }
-    const blockStart = index * this.getBlockSize()
-    const blockEnd = (index + 1) * this.getBlockSize()
-    return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && offset < blockEnd)
+
+    const blockSize = this.getBlockSize()
+    const blockStart = index * blockSize
+    const blockEnd = blockStart + blockSize
+
+    let startExtentIndex = 0
+    if (this.#hasBlockCursor !== undefined && index >= this.#hasBlockPreviousIndex) {
+      startExtentIndex = this.#hasBlockCursor
+    } else {
+      this.#hasBlockCursor = undefined
+    }
+    this.#hasBlockPreviousIndex = index
+
+    const dataMap = this.#dataMap
+    const l = dataMap.length
+    for (let i = startExtentIndex; i < l; i++) {
+      const { offset, length } = dataMap[i]
+      if (offset >= blockEnd) {
+        // extents are sorted: nothing from here on can overlap this block
+        break
+      }
+      if (offset + length > blockStart) {
+        this.#hasBlockCursor = i
+        return true
+      }
+    }
+    return false
   }
 }

@vates/nbd-client/NbdDisk.test.mjs

@@ -0,0 +1,127 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
+
+import { NbdDisk } from './NbdDisk.mjs'
+
+const KiB = 1024
+const MiB = 1024 * KiB
+const TiB = 1024 * 1024 * MiB
+
+const SOURCE_BLOCK_SIZE = 2 * MiB // VHD_BLOCK_SIZE, the block size used when importing from VMware
+const CLUSTER_SIZE = 64 * KiB // qcow2 cluster size
+
+// hasBlock() only needs the dataMap (passed through the constructor) and the
+// block size, so we can exercise it without a live NBD connection.
+describe('NbdDisk.hasBlock', () => {
+  it('reports allocated and unallocated blocks correctly', () => {
+    const dataMap = [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
+      { type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
+    ]
+    const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+
+    assert.strictEqual(disk.hasBlock(0), true)
+    assert.strictEqual(disk.hasBlock(1), false)
+    assert.strictEqual(disk.hasBlock(4), false)
+    assert.strictEqual(disk.hasBlock(5), true)
+    assert.strictEqual(disk.hasBlock(6), false)
+  })
+
+  it('handles repeated and out-of-order queries with the resume cursor', () => {
+    const dataMap = [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
+      { type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
+    ]
+    const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+
+    // repeated indexes (the qcow2 path queries each 2MB block ~32 times as it
+    // walks 64KB clusters) must return a stable answer
+    assert.strictEqual(disk.hasBlock(5), true)
+    assert.strictEqual(disk.hasBlock(5), true)
+    assert.strictEqual(disk.hasBlock(6), false)
+    assert.strictEqual(disk.hasBlock(6), false)
+
+    // querying backwards must still be correct (falls back to a full scan)
+    assert.strictEqual(disk.hasBlock(0), true)
+    assert.strictEqual(disk.hasBlock(5), true)
+  })
+
+  it('invalidates the cursor on a backward miss (no stale skip)', () => {
+    const dataMap = [
+      { type: 0, offset: 3 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 3
+      { type: 0, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 10
+    ]
+    const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+
+    assert.strictEqual(disk.hasBlock(10), true) // forward hit: cursor parks on the high extent
+    assert.strictEqual(disk.hasBlock(1), false) // backward miss: must invalidate the cursor
+    assert.strictEqual(disk.hasBlock(3), true) // lower extent must not be skipped
+  })
+
+  it('throws on overlapping ranges (would break the resume cursor)', () => {
+    const dataMap = [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE },
+      { type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE }, // overlaps the previous range
+    ]
+    assert.throws(() => new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap }), /overlapping ranges/)
+  })
+
+  it('merges touching ranges and drops zero-length / non type-0 entries', () => {
+    const dataMap = [
+      { type: 0, offset: 2 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [2, 3)
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5)
+      { type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [1, 2)
+      { type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 }, // [.5, 1), touches [0, .5)
+      { type: 1, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // wrong type: ignored
+      { type: 0, offset: 7 * SOURCE_BLOCK_SIZE, length: 0 }, // zero length: ignored
+    ]
+    const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+
+    assert.strictEqual(disk.hasBlock(0), true)
+    assert.strictEqual(disk.hasBlock(1), true)
+    assert.strictEqual(disk.hasBlock(2), true)
+    assert.strictEqual(disk.hasBlock(3), false)
+    assert.strictEqual(disk.hasBlock(7), false) // zero-length range dropped
+    assert.strictEqual(disk.hasBlock(10), false) // non type-0 range dropped
+  })
+})
+
+describe('NbdDisk.getBlockIndexesCount', () => {
+  const cases = {
+    'aligned, disjoint blocks': [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
+      { type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
+    ],
+    'a range spanning several blocks': [
+      { type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: 3 * SOURCE_BLOCK_SIZE }, // blocks 0,1,2,3
+    ],
+    'two disjoint extents sharing block 0 (needs dedup)': [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5) => block 0
+      { type: 0, offset: 3 * (SOURCE_BLOCK_SIZE / 4), length: SOURCE_BLOCK_SIZE }, // [.75, 1.75) => blocks 0,1
+    ],
+    'touching sub-block fragments': [
+      { type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 },
+      { type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 },
+      { type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE },
+    ],
+    empty: [],
+  }
+
+  for (const [name, dataMap] of Object.entries(cases)) {
+    it(`matches getBlockIndexes().length: ${name}`, () => {
+      const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+      assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
+    })
+  }
+
+  it('matches getBlockIndexes().length on the large sparse map', () => {
+    const virtualSize = Math.floor(3.5 * TiB)
+    const NB_RANGES = 113528
+    const dataMap = []
+    for (let i = 0; i < NB_RANGES; i++) {
+      dataMap.push({ type: 0, offset: Math.floor((i / NB_RANGES) * virtualSize), length: CLUSTER_SIZE })
+    }
+    const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
+    assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
+  })
+})

@vates/nbd-client/package.json

@@ -13,7 +13,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "engines": {
     "node": ">=14.0"
   },
@@ -22,10 +22,10 @@
     "@vates/async-each": "^1.0.3",
     "@vates/read-chunk": "^1.2.1",
     "@xen-orchestra/async-map": "^0.1.3",
-    "@xen-orchestra/disk-transform": "^1.2.3",
+    "@xen-orchestra/disk-transform": "^1.3.3",
     "@xen-orchestra/log": "^0.7.2",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^4.7.7"
+    "xen-api": "^4.8.0"
   },
   "devDependencies": {
     "tap": "^18.7.0",

@vates/types/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vates/types",
   "private": false,
-  "version": "1.23.0",
+  "version": "1.30.1",
   "main": "./dist/index.mjs",
   "exports": {
     ".": {
@@ -32,7 +32,8 @@
   "devDependencies": {
     "@eslint/js": "^9.19.0",
     "typescript": "~5.6.3",
-    "typescript-eslint": "^8.23.0"
+    "typescript-eslint": "^8.61.0",
+    "@types/express": "^5.0.0"
   },
   "scripts": {
     "build": "tsc",

@vates/types/src/common.mts

@@ -1,4 +1,4 @@
-import type { XoHost } from './xo.mjs'
+import type { XoHost, XoNetwork, XoVif } from './xo.mjs'
 
 declare const __brand: unique symbol
 
@@ -740,3 +740,32 @@ export type XapiVmStatsRaw = {
 export type XapiVmStats = XapiStatsResponse<XapiVmStatsRaw>
 
 export type XapiPoolStats = Record<XoHost['id'], XapiHostStats | { error: Record<string, unknown> }>
+
+export type TrafficRuleProtocol = (typeof TRAFFIC_RULE_PROTOCOLS)[number]
+
+export const SDN_CONTROLLER_OF_RULES_KEY = 'xo:sdn-controller:of-rules'
+export const SDN_CONTROLLER_OF_METHOD_KEY = 'xo:sdn-controller:of-method'
+export const SDN_CONTROLLER_OF_FORMAT_KEY = 'xo:sdn-controller:of-format'
+export const TRAFFIC_RULE_PROTOCOLS = ['ARP', 'ICMP', 'IP', 'TCP', 'UDP'] as const
+export const TRAFFIC_RULE_PROTOCOLS_WITH_PORT: readonly TrafficRuleProtocol[] = ['TCP', 'UDP']
+
+export type TrafficRuleTargetType = XoNetwork['type'] | XoVif['type']
+
+export type TrafficRuleDirection = 'from' | 'to' | 'from/to'
+
+export type RawTrafficRule = {
+  allow: boolean
+  protocol: string
+  ipRange: string
+  direction: string
+  port?: string
+}
+
+type BaseTrafficRule = RawTrafficRule & {
+  id: string
+  networkId: XoNetwork['id']
+}
+
+export type TrafficRule =
+  | (BaseTrafficRule & { type: XoVif['type']; sourceId: XoVif['id'] })
+  | (BaseTrafficRule & { type: XoNetwork['type']; sourceId: XoNetwork['id'] })

@vates/types/src/index.mts

@@ -3,4 +3,6 @@ export * from './xen-api.mjs'
 export * from './xo.mjs'
 export * from './xo-app.mjs'
 export * from './lib/xen-orchestra-xapi.mjs'
+export * from './lib/xen-orchestra-acl.mjs'
 export * from './lib/complex-matcher.mjs'
+export * from './lib/rest-api.mjs'

@vates/types/src/lib/rest-api.mts

@@ -0,0 +1,102 @@
+import type { NextFunction, Request, Response } from 'express'
+import type { VatesTask } from './vates-task.mjs'
+import type { XoApp } from '../xo-app.mjs'
+
+export type SecurityName = '*' | 'token' | 'basic' | 'none'
+
+export type FieldDefinition =
+  | {
+      type: 'string'
+      example?: string
+      optional?: boolean
+    }
+  | {
+      type: 'boolean'
+      example?: boolean
+      optional?: boolean
+    }
+  | {
+      type: 'number'
+      example?: number
+      optional?: boolean
+    }
+  | {
+      type: 'enum'
+      enum: string[]
+      example?: string
+      optional?: boolean
+    }
+  | {
+      type: 'object'
+      fields: Record<string, FieldDefinition>
+      optional?: boolean
+    }
+  | {
+      type: 'array'
+      items: FieldDefinition
+      example?: unknown[]
+      optional?: boolean
+    }
+
+export type ParamFieldDefinition = Exclude<
+  FieldDefinition,
+  { type: 'boolean' } | { type: 'object' } | { type: 'array' }
+>
+
+export type QueryFieldDefinition = Exclude<FieldDefinition, { type: 'object' } | { type: 'array' }>
+type RestApi = object & { xoApp: XoApp }
+export type CreateAction = <Result>(
+  callback: (task: VatesTask) => MaybePromise<Result>,
+  opts: {
+    sync?: boolean
+    statusCode?: number
+    taskProperties: { name: string; [key: string]: unknown }
+  }
+) => Promise<Result | undefined>
+
+export interface BaseRouteDefinition<Middleware> {
+  method: 'get' | 'post' | 'put' | 'delete' | 'patch'
+  endpoint: string
+  description?: string
+  tags?: string[]
+  params?: Record<string, ParamFieldDefinition>
+  query?: Record<string, QueryFieldDefinition>
+  body?: Record<string, FieldDefinition>
+  responses?: Array<{
+    status: number
+    description: string
+    schema?: Record<string, FieldDefinition>
+  }>
+  middlewares?: Middleware[]
+  scope?: 'acl'
+  callback: (params: {
+    req: Request
+    res: Response
+    next: NextFunction
+    restApi: RestApi
+    createAction: CreateAction
+  }) => MaybePromise<unknown>
+  security?: SecurityName
+}
+
+export type MiddlewareDescriptor =
+  | { name: 'json' | 'urlencoded' | 'text' | 'raw'; options?: Record<string, unknown> }
+  | { name: 'acl'; acls: LooseAclEntry | LooseAclEntry[] }
+
+type MaybePromise<T> = T | Promise<T>
+/**
+ * Loosely-typed mirror of {@link https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts | AclEntry}
+ * from `@xen-orchestra/rest-api/src/middlewares/acl.middleware.mts`.
+ */
+export type LooseAclEntry = {
+  resource: string
+  action?: string | ((opts: { req: object; restApi: RestApi }) => string | undefined)
+  actions?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
+  objectId?: string | ((opts: { req: object; restApi: RestApi }) => string)
+  objectIds?: string[] | ((opts: { req: object; restApi: RestApi }) => string[])
+  object?: object | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object> | undefined)
+  objects?: object[] | ((opts: { req: object; restApi: RestApi }) => MaybePromise<object[]> | undefined)
+  getObject?: (opts: { restApi: RestApi }) => (id: string) => MaybePromise<object>
+}
+
+export type PluginRestRouteDefinition = BaseRouteDefinition<MiddlewareDescriptor>

@vates/types/src/lib/xen-orchestra-acl.mts

@@ -0,0 +1,83 @@
+import { Branded } from '../common.mjs'
+import type { XoGroup, XoUser } from '../xo.mjs'
+
+type XoAclBaseRole = {
+  id: Branded<'acl-v2-role'>
+  name: string
+  description?: string
+  privilegeIds: XoAclBasePrivilege['id'][]
+}
+
+type XoAclRegularRole = XoAclBaseRole & {
+  groupIds: XoGroup['id'][]
+  userIds: XoUser['id'][]
+}
+
+type XoAclTemplateRole = XoAclBaseRole & {
+  isTemplate: true
+  roleTemplateId: number
+}
+
+export type XoAclRole = XoAclRegularRole | XoAclTemplateRole
+
+export type XoAclSupportedActionsByResource = {
+  [resource: string]: Record<string, unknown>
+}
+
+/**
+ * E.g
+ * vms {
+ *   shutdown: {
+ *    clean: true,
+ *    hard: true,
+ *  }
+ * }
+ *
+ * `shutdown | shutdown:clean | shutdown:hard`
+ */
+export type GetKeysRecursively<T, Prefix extends string = ''> = {
+  [K in keyof T]: T[K] extends object
+    ? K extends string
+      ? `${Prefix}${K}` | GetKeysRecursively<T[K], `${Prefix}${K}:`>
+      : never
+    : K extends string
+      ? `${Prefix}${K}`
+      : never
+}[keyof T]
+
+export type XoAclSupportedResource<TActionsByResource extends XoAclSupportedActionsByResource> =
+  keyof TActionsByResource
+
+export type XoAclSupportedActions<
+  TActionsByResource extends XoAclSupportedActionsByResource,
+  TResource extends XoAclSupportedResource<TActionsByResource>,
+> = (GetKeysRecursively<TActionsByResource[TResource]> & string) | '*'
+
+export type XoAclBasePrivilege = {
+  id: Branded<'acl-v2-privilege'>
+  resource: string
+  action: string
+  selector?: string
+  effect: 'allow' | 'deny'
+  roleId: XoAclRole['id']
+}
+
+export type XoAclPrivilege<
+  TActionsByResource extends XoAclSupportedActionsByResource,
+  TResource extends XoAclSupportedResource<TActionsByResource>,
+> = XoAclBasePrivilege & {
+  resource: TResource
+  action: XoAclSupportedActions<TActionsByResource, TResource>
+}
+
+export type XoUserRole = {
+  id: string
+  roleId: XoAclRole['id']
+  userId: XoUser['id']
+}
+
+export type XoGroupRole = {
+  id: string
+  roleId: XoAclRole['id']
+  groupId: XoGroup['id']
+}

@vates/types/src/lib/xen-orchestra-xapi.mts

@@ -22,9 +22,11 @@ import type {
   VBD_TYPE,
   VDI_TYPE,
   VIF_LOCKING_MODE,
+  VM_OPERATIONS,
 } from '../common.mjs'
 import type { PassThrough, Readable } from 'node:stream'
 import type {
+  XapiXoRecord,
   XoGpuGroup,
   XoHost,
   XoNetwork,
@@ -36,8 +38,67 @@ import type {
   XoVm,
   XoVmTemplate,
   XoVif,
+  XoVmSnapshot,
 } from '../xo.mjs'
 
+/**
+ * Properties accepted by {@link Xapi.editVm}.
+ *
+ * Field names match the canonical (camelCase) properties defined by the
+ * underlying `_editVm` (via `makeEditObject`). `_editVm` still resolves the
+ * snake_case and legacy aliases, but the canonical names are used here.
+ */
+export interface EditVmProps {
+  affinityHost?: string | null
+  autoPoweron?: boolean
+  blockedOperations?: Partial<Record<VM_OPERATIONS, boolean | string | null>>
+  coresPerSocket?: number | string | null
+  cpuCap?: number | null
+  cpuMask?: number[]
+  cpuWeight?: number | null
+  cpus?: number
+  cpusStaticMax?: number | string
+  /**
+   * Update VM creation metadata stored under `other_config.xo:*`. The object is
+   * merged with the existing data.
+   */
+  creation?: { user?: string }
+  expNestedHvm?: boolean
+  hasVendorDevice?: boolean
+  highAvailability?: 'best-effort' | 'restart' | ''
+  hvmBootFirmware?: string | null
+  memory?: number | string
+  memoryMax?: number | string
+  memoryMin?: number | string
+  memoryStaticMax?: number | string
+  nameDescription?: string
+  nameLabel?: string
+  nestedVirt?: boolean
+  nicType?: string | null
+  notes?: string | null
+  PV_args?: string
+  secureBoot?: boolean
+  startDelay?: number
+  suspendSr?: string | null
+  tags?: string[]
+  uefiMode?: 'setup' | 'user'
+  vga?: 'std' | 'cirrus'
+  videoram?: 1 | 2 | 4 | 8 | 16
+  viridian?: boolean
+  virtualizationMode?: 'pv' | 'hvm'
+  xenStoreData?: Record<string, string | null>
+}
+
+/** Properties accepted by {@link Xapi.editVif}. */
+export interface EditVifProps {
+  ipv4Allowed?: string[]
+  ipv6Allowed?: string[]
+  lockingMode?: VIF_LOCKING_MODE
+  // in kB/s, `null` removes the limit
+  rateLimit?: number | null
+  txChecksumming?: boolean
+}
+
 export type XcpPatches = {
   changelog?: {
     author: string
@@ -68,16 +129,29 @@ export type XsPatches = {
 }
 
 export interface Xapi {
+  status: string
+  pool?: { uuid: string }
+  sessionId: string
+  _auth: { user: string; password: string }
+  _url: { protocol: string; hostname: string; hostnameRaw: string; port?: string }
+
   call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
   callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
 
   barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
+  connectVif(vifId: XoVif['id']): Promise<void>
   getField<T extends XenApiRecord, K extends keyof T>(
     type: Extract<WrappedXenApiRecord, T>['$type'],
     ref: T['$ref'],
     field: K
   ): Promise<T[K]>
-  connectVif(vifId: XoVif['id']): Promise<void>
+  getObject: <
+    XoRecord extends XapiXoRecord,
+    WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
+  >(
+    idOrUuidOrRef: XoRecord['id'] | WrappedRecord['$ref'] | WrappedRecord['uuid'],
+    defaultValue?: WrappedRecord
+  ) => WrappedRecord
   createNetwork(
     params:
       | {
@@ -105,6 +179,13 @@ export interface Xapi {
   deleteVif(vifId: XoVif['id']): Promise<void>
   disconnectVif(vifId: XoVif['id']): Promise<void>
   exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
+  emergencyShutdownHost(hostId: XoHost['id']): Promise<void>
+  ejectHostFromPool(hostId: XoHost['id']): Promise<void>
+  host_smartReboot(
+    hostRef: XenApiHost['$ref'],
+    bypassBlockedSuspend?: boolean,
+    bypassCurrentVmCheck?: boolean
+  ): Promise<void>
   migrateVm(
     vmId: XoVm['id'],
     hostXapi: Xapi,
@@ -118,9 +199,11 @@ export interface Xapi {
       bypassAssert?: boolean
     }
   ): Promise<void>
+  joinPool(masterAddress: string, masterUsername: string, masterPassword: string, force?: boolean): Promise<void>
   listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
   pool_emergencyShutdown(): Promise<void>
   resumeVm(id: XoVm['id']): Promise<void>
+  revertVm(snapshotId: XoVmSnapshot['id']): Promise<void>
   unpauseVm(id: XoVm['id']): Promise<void>
   cloneVm(
     vmId: XoVm['id'],
@@ -146,6 +229,40 @@ export interface Xapi {
     }
   ): Promise<{ vm: XenApiVmWrapped }>
   forgetSr(id: XoSr['id']): Promise<void>
+  forgetHost(hostId: XoHost['id']): Promise<void>
+  powerOnHost(hostId: XoHost['id']): Promise<void>
+  rebootHost(hostId: XoHost['id'], force?: boolean): Promise<void>
+  shutdownHost(hostId: XoHost['id'], opts?: { force?: boolean; bypassEvacuate?: boolean }): Promise<void>
+  SR_create(params: {
+    content_type?: XoSr['content_type']
+    // index signature, not `Record<string, string>`: tsoa emits an empty schema for `Record`
+    // (https://github.com/lukeautry/tsoa/pull/1858). device_config keys are driver-specific (SR_type).
+    device_config: {
+      chapuser?: string
+      chappassword?: string
+      device?: string
+      legacy_mode?: string
+      location?: string
+      nfsversion?: string
+      options?: string
+      password?: string
+      SCSIid?: string
+      server?: string
+      serverpath?: string
+      target?: string
+      targetIQN?: string
+      username?: string
+    } & {
+      [key: string]: string
+    }
+    host: XenApiHost['$ref']
+    name_description?: XoSr['name_description']
+    name_label: XoSr['name_label']
+    physical_size?: number
+    shared: boolean
+    sm_config?: { [key: string]: string }
+    type: XoSr['SR_type']
+  }): Promise<XenApiSr['$ref']>
   SR_importVdi(
     ref: XenApiSr['$ref'],
     stream: Readable,
@@ -164,6 +281,12 @@ export interface Xapi {
     }
   ): Promise<XenApiVdi['$ref']>
   SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
+  editVm(
+    id: XoVm['id'],
+    props: EditVmProps,
+    checkLimits?: (limits: Record<string, number>, vm: XenApiVmWrapped) => Promise<void>
+  ): Promise<void>
+  editVif(id: XoVif['id'], props: EditVifProps): Promise<void>
   startVm(
     id: XoVm['id'],
     opts?: {
@@ -181,7 +304,7 @@ export interface Xapi {
   ): Promise<void>
   VM_export(
     vmRef: XenApiVm['$ref'],
-    opts?: { cancelToken?: unknown; compress?: boolean; useSnapshot?: boolean }
+    opts?: { cancelToken?: unknown; compress?: boolean | 'zstd' | 'gzip'; useSnapshot?: boolean }
   ): ReturnType<Xapi['getResource']>
   VM_import(
     stream: Readable,
@@ -235,6 +358,7 @@ export interface Xapi {
       vgpuType?: XoVgpuType['id']
       gpuGroup?: XoGpuGroup['id']
       copyHostBiosStrings?: boolean
+      highAvailability?: XoVm['high_availability']
       hvmBootFirmware?: 'uefi' | 'bios'
       secureBoot?: boolean
     },
@@ -287,7 +411,7 @@ export interface Xapi {
   ): Promise<Readable & { length?: number }>
   VDI_importContent(
     vdiRef: XenApiVdi['$ref'],
-    stream: Readable,
+    stream: Readable | Buffer,
     opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
   ): Promise<void>
   VIF_create(
@@ -326,8 +450,12 @@ export interface Xapi {
     pathname: string,
     params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
   ): Promise<{ body: Readable }>
-  clearHost(host: Pick<XenApiHostWrapped, '$ref' | '$pool'>, force?: boolean): Promise<void>
-  disableHost(hostId: XoHost['id']): Promise<void>
+  clearHost(
+    host: Pick<XenApiHostWrapped, '$ref' | '$pool'>,
+    force?: boolean,
+    opts?: { transient?: boolean }
+  ): Promise<void>
+  disableHost(hostId: XoHost['id'], opts?: { transient?: boolean }): Promise<void>
   enableHost(hostId: XoHost['id']): Promise<void>
   getRecordByUuid<
     Type extends WrappedXenApiRecord['$type'],
@@ -338,4 +466,13 @@ export interface Xapi {
   ): Promise<XenApiRecord>
   isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
   VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
+  destroySr(id: XoSr['id']): Promise<void>
+  xostor_delete(ref: XenApiSr['$ref']): Promise<void>
+  objects: {
+    indexes: {
+      type: {
+        [XenApiRecord in WrappedXenApiRecord as XenApiRecord['$type']]: Record<XenApiRecord['uuid'], XenApiRecord>
+      }
+    }
+  }
 }

@vates/types/src/xen-api.mts

@@ -192,6 +192,7 @@ export interface XenApiPool {
   ha_host_failures_to_tolerate: number
   ha_overcommitted: boolean
   ha_plan_exists_for: number
+  ha_reboot_vm_on_internal_shutdown?: boolean
   ha_statefiles: string[]
   health_check_config: Record<string, string>
   igmp_snooping_enabled?: boolean
@@ -491,7 +492,9 @@ export interface XenApiDrTask {
   uuid: string
 }
 
-type XenApiHostCallMethods = TagCallMethods & {}
+type XenApiHostCallMethods = TagCallMethods & {
+  <T>(method: 'call_plugin', plugin: string, fn: string, args: Record<string, string>): Promise<T>
+}
 export interface XenApiHost {
   $ref: Branded<'host'>
   address: string
@@ -568,7 +571,9 @@ export interface XenApiHost {
   uuid: string
   virtual_hardware_platform_versions: number[]
 }
-export type XenApiHostWrapped = WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods>
+export interface XenApiHostWrapped extends WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods> {
+  $restartAgent(): Promise<void>
+}
 
 export interface XenApiHostCrashdump {
   $ref: Branded<'host_crashdump'>

@vates/types/src/xo-app.mts

@@ -1,6 +1,5 @@
 import { EventEmitter } from 'node:stream'
 import type {
-  AnyXoBackupJob,
   AnyXoJob,
   AnyXoLog,
   XapiXoRecord,
@@ -16,10 +15,12 @@ import type {
   XoServer,
   XoTask,
   XoUser,
+  XoVif,
   XoVm,
   XoVmBackupArchive,
 } from './xo.mjs'
 import { VatesTask } from './lib/vates-task.mjs'
+import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
 import {
   Xapi,
   XapiHostStats,
@@ -44,14 +45,32 @@ import {
   XenApiVifWrapped,
   XenApiVmWrapped,
   XenApiVtpmWrapped,
+  XoAclBasePrivilege,
+  XoAclRole,
+  XoGroupRole,
+  XoUserRole,
 } from './index.mjs'
 
-export type XapiConnection = Xapi & {
-  status: string
-  pool?: { uuid: string }
-  sessionId: string
-  _url?: { protocol: string; hostname: string; port?: string }
-}
+type FeatureCode =
+  | 'BACKUP.DELTA'
+  | 'BACKUP.DELTA_REPLICATION'
+  | 'BACKUP.FULL'
+  | 'BACKUP.HEALTHCHECK'
+  | 'BACKUP.METADATA'
+  | 'BACKUP.MIRROR'
+  | 'BACKUP.WITH_RAM'
+  | 'BACKUP.SMART_BACKUP'
+  | 'BACKUP.S3'
+  | 'DOCKER'
+  | 'EXPORT.XVA'
+  | 'LIST_MISSING_PATCHES'
+  | 'POOL_EMERGENCY_SHUTDOWN'
+  | 'RBAC'
+  | 'ROLLING_POOL_UPDATE'
+  | 'ROLLING_POOL_REBOOT'
+  | 'SMART_REBOOT'
+  | 'WARM_MIGRATION'
+  | 'PLUGIN.OPENMETRICS'
 
 type XapiRecordByXapiXoRecord = {
   gpuGroup: XenApiGpuGroupWrapped
@@ -78,10 +97,45 @@ type XapiRecordByXapiXoRecord = {
   'VM-template': XenApiVmWrapped
   VTPM: XenApiVtpmWrapped
 }
+
+type LicenseProductId =
+  | 'premium'
+  | 'xcpng-enterprise'
+  | 'xcpng-standard'
+  | 'xo-proxy'
+  | 'xosan.trial'
+  | 'xostor'
+  | 'xostor.trial'
+type LicenseProductType = 'xo' | 'xoproxy' | 'xcpng' | 'xosan' | 'xostor'
+
+type License = {
+  id: string
+  type: 'license'
+  created: number
+  licenseType: 'unit'
+  productId: LicenseProductId
+  tags: string[]
+  expires?: number
+  boundObjectId?: string
+  bindDate?: number
+  buyer?: { token: string; email: string }
+  history?: { date: number; boundObjectId: string }[]
+  rebound?: number
+  productTypes?: LicenseProductType[]
+  bundleInfo?: { name: string; id: string }
+}
+
 export type XoApp = {
+  hooks: EventEmitter
+  _redis: {
+    get(key: string): Promise<string | null>
+    mSet(args: string[]): Promise<unknown>
+    sMembers(key: string): Promise<string[]>
+    sIsMember(key: string, member: string): Promise<boolean>
+  }
   config: {
     get<T = string>(path: string): T
-    getOptional(path: string): Record<string, string> | undefined
+    getOptional<T = unknown>(path: string): T | undefined
     getOptionalDuration(path: string): number | undefined
     getGuiRoutes(): Promise<{
       default: {
@@ -128,16 +182,42 @@ export type XoApp = {
   }
 
   // methods ------------
+  allocIpAddresses(vifId: XoVif['id'], addAddresses?: string[], removeAddresses?: string[]): Promise<void>
+  addAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<XoGroupRole>
+  addAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<XoUserRole>
   addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
+  addApiMethod: <A extends unknown[], R>(
+    name: string,
+    method: (...args: A) => Promise<R>,
+    //we use any since it is a legacy call, might be better typed in another pr
+    info: {
+      resolve?: any
+      params?: any
+    }
+  ) => () => void // eslint-disable-line @typescript-eslint/no-explicit-any
+  registerRestRoutes: (routes: PluginRestRouteDefinition[], base?: string) => () => void
   authenticateUser: (
     credentials: { token?: string; username?: string; password?: string },
     userData?: { ip?: string },
     opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
   ) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
+  backupGuard(poolId: XoPool['id']): Promise<void>
   /* Throw if no authorization */
-  checkFeatureAuthorization(featureCode: string): Promise<void>
+  checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
   /* connect a server (XCP-ng/XenServer) */
   connectXenServer(id: XoServer['id']): Promise<void>
+  // TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)
+  createAclV2Privilege(
+    privilege: Omit<XoAclBasePrivilege, 'id'>,
+    options?: {
+      force?: boolean
+    }
+  ): Promise<XoAclBasePrivilege>
+  copyAclV2Role(
+    id: XoAclRole['id'],
+    params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
+  ): Promise<XoAclRole['id']>
+  createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
   createAuthenticationToken(opts: {
     client?: {
       id?: string
@@ -147,19 +227,54 @@ export type XoApp = {
     expiresIn?: string | number
     userId: XoUser['id']
   }): Promise<XoAuthenticationToken>
+  createRemote(params: {
+    name: string
+    options?: string
+    proxy?: XoProxy['id']
+    url: string
+  }): Promise<XoBackupRepository>
   createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
+  deleteAclV2GroupRole(
+    groupId: XoGroup['id'],
+    roleId: XoAclRole['id'],
+    opts?: { bypassAuthorization?: boolean }
+  ): Promise<boolean>
+  deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
+  deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
   deleteGroup(id: XoGroup['id']): Promise<void>
   deleteUser(id: XoUser['id']): Promise<void>
+  detachHostFromPool(hostId: XoHost['id']): Promise<void>
   createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
   /* disconnect a server (XCP-ng/XenServer) */
   disconnectXenServer(id: XoServer['id']): Promise<void>
+  findEnabledScheduleSequenceFromSchedule(id: XoSchedule['id']): Promise<XoSchedule | undefined>
+  getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
+  getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
+  getAclV2RolePrivileges(roleId: XoAclRole['id']): Promise<XoAclBasePrivilege[]>
+  getAclV2Role(id: XoAclRole['id'], opts?: { bypassAuthorization?: boolean }): Promise<XoAclRole>
+  deleteAclV2UserRole(
+    userId: XoUser['id'],
+    roleId: XoAclRole['id'],
+    opts?: { bypassAuthorization?: boolean }
+  ): Promise<boolean>
+  getAclV2GroupRoles(
+    groupId: XoGroup['id'],
+    opts?: { bypassAuthorization?: boolean }
+  ): Promise<Exclude<XoAclRole, { isTemplate: true }>[]>
+  getAclV2Roles(): Promise<XoAclRole[]>
+  getAclV2UserPrivileges(userId: XoUser['id']): Promise<XoAclBasePrivilege[]>
+  getAclV2UserRoles(
+    userId: XoUser['id'],
+    opts?: { bypassAuthorization?: boolean; fromGroup?: boolean; fromUser?: boolean }
+  ): Promise<XoAclRole[]>
   getAllGroups(): Promise<XoGroup[]>
   getAllProxies(): Promise<XoProxy[]>
-  getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
+  getAllJobs<T extends AnyXoJob['type']>(type: T): Promise<Extract<AnyXoJob, { type: T }>[]>
   getAllJobs(type?: string): Promise<AnyXoJob[]>
   getProxy(id: XoProxy['id']): Promise<XoProxy>
   getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
   getAllRemotes(): Promise<XoBackupRepository[]>
+  removeRemote(id: XoBackupRepository['id']): Promise<void>
   getAllRemotesInfo(): Promise<
     Record<
       XoBackupRepository['id'],
@@ -217,6 +332,7 @@ export type XoApp = {
     backupRepositoryIds: XoBackupRepository['id'][],
     opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
   ): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
+  pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
   /** Allow to add a new server in the DB (XCP-ng/XenServer) */
   registerXenServer(
     body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
@@ -225,11 +341,22 @@ export type XoApp = {
       readOnly?: XoServer['readOnly']
     }
   ): Promise<XoServer>
-  rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise<void>
-  rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise<void>
+  rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask; shutdownPinnedVms?: boolean }): Promise<void>
+  rollingPoolUpdate(
+    pool: XoPool,
+    opts?: { rebootVm?: boolean; parentTask?: VatesTask; shutdownPinnedVms?: boolean }
+  ): Promise<void>
+  setVmResourceSet(vmId: XoVm['id'], resourceSetId: string | null, force?: boolean): Promise<void>
+  shareVmResourceSet(vmId: XoVm['id']): Promise<void>
   removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
   runJob(job: AnyXoJob, schedule: XoSchedule): void
   runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
+  testRemote(
+    id: XoBackupRepository['id']
+  ): Promise<
+    | { success: true; readRate: number; writeRate: number }
+    | { success: false; step: string; file: string; error: unknown }
+  >
   /** Remove a server from the DB (XCP-ng/XenServer) */
   unregisterXenServer(id: XoServer['id']): Promise<void>
   updateUser(
@@ -246,12 +373,44 @@ export type XoApp = {
       preferences?: Record<string, string>
     }
   ): Promise<void>
+  updateAclV2Privilege(
+    privilegeId: XoAclBasePrivilege['id'],
+    privilege: XoAclBasePrivilege
+  ): Promise<XoAclBasePrivilege>
+  updateAclV2Role(
+    roleId: XoAclRole['id'],
+    role: {
+      name?: XoAclRole['name']
+      description?: XoAclRole['description'] | null
+    },
+    options?: {
+      force?: boolean
+    }
+  ): Promise<XoAclRole>
   updateGroup(
     id: XoGroup['id'],
     updates: {
       name?: string
     }
   ): void
-  getAllXapis(): Record<string, XapiConnection>
+  updateRemote(
+    id: XoBackupRepository['id'],
+    params: {
+      enabled?: boolean
+      name?: string
+      options?: string | null
+      proxy?: XoProxy['id'] | null
+      url?: string
+    }
+  ): Promise<XoBackupRepository>
+  getAllXapis(): Record<string, Xapi>
   getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
+  getLicenses(params?: { productType?: LicenseProductType }): Promise<License[]>
+  bindLicense(params: { licenseId: string; boundObjectId: string }): Promise<License>
+  unbindLicense(params: {
+    productId: LicenseProductId
+    boundObjectId: string
+    licenseId: string
+    data?: Record<string, string>
+  }): Promise<License>
 }

@vates/types/src/xo.mts

@@ -19,6 +19,7 @@ import type {
   VM_POWER_STATE,
 } from './common.mjs'
 import type * as CMType from './lib/complex-matcher.mjs'
+import { XoAclPrivilege, XoAclRole, XoAclSupportedActionsByResource } from './lib/xen-orchestra-acl.mjs'
 import type { XenApiHost, XenApiPool } from './xen-api.mjs'
 
 type BaseXapiXo = {
@@ -75,7 +76,7 @@ type BaseXoVm = BaseXapiXo & {
   expNestedHvm: boolean
   isNestedVirtEnabled: boolean
   hasVendorDevice: boolean
-  high_availability: string
+  high_availability: 'best-effort' | 'restart' | ''
   installTime?: number | null
   isFirmwareSupported: boolean
   memory: {
@@ -221,6 +222,7 @@ export type XoGroup = {
   provider?: string
   providerGroupId?: string
   users: XoUser['id'][]
+  aclRoleIds: XoAclRole['id'][]
 }
 
 export type XoHost = BaseXapiXo & {
@@ -334,6 +336,7 @@ export type XoNetwork = BaseXapiXo & {
   defaultIsLocked: boolean
   id: Branded<'network'>
   insecureNbd?: boolean
+  isBonded: boolean
   name_description: string
   name_label: string
   nbd?: boolean
@@ -431,6 +434,7 @@ export type XoPool = BaseXapiXo & {
   current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
   default_SR?: XoSr['id']
   HA_enabled: boolean
+  haRebootVmOnInternalShutdown: boolean
   haSrs: XoSr['id'][]
   id: Branded<'pool'>
   master: XoHost['id']
@@ -562,6 +566,18 @@ export type XoMirrorBackupJob = BaseXoJob & {
 
 export type XoJob = BaseXoJob & {
   type: 'call'
+  method: string
+  paramsVector?: {
+    type: string
+    items: {
+      type?: string
+      values?: {
+        schedules?: XoSchedule['id'][]
+        [key: string]: unknown
+      }[]
+      [key: string]: unknown
+    }[]
+  }
 }
 
 export type XoSchedule = {
@@ -598,7 +614,7 @@ export type XoSr = BaseXapiXo & {
   VDIs: AnyXoVdi['id'][]
 
   allocationStrategy?: 'thin' | 'thick' | 'unknown'
-  content_type: string
+  content_type: 'user' | 'iso' | 'disk'
   current_operations: Record<string, STORAGE_OPERATIONS>
   id: Branded<'SR'>
   inMaintenanceMode: boolean
@@ -609,7 +625,28 @@ export type XoSr = BaseXapiXo & {
   shared: boolean
   size: number
   sm_config: Record<string, string>
-  SR_type: string
+  // SM drivers shipped with XCP-ng/XenServer
+  SR_type:
+    | 'cephfs'
+    | 'dummy'
+    | 'ext'
+    | 'file'
+    | 'glusterfs'
+    | 'hba'
+    | 'iso'
+    | 'largeblock'
+    | 'linstor'
+    | 'lvm'
+    | 'lvmofcoe'
+    | 'lvmohba'
+    | 'lvmoiscsi'
+    | 'moosefs'
+    | 'nfs'
+    | 'shm'
+    | 'smb'
+    | 'udev'
+    | 'xfs'
+    | 'zfs'
   tags: string[]
   type: 'SR'
   usage: number
@@ -656,13 +693,15 @@ export type XoTask = {
       | 'server'
       | 'task'
       | 'user'
+      | 'acl-privilege'
+      | 'acl-role'
     params?: Record<string, unknown>
     progress?: number
     type?: string
     userId?: string
     [key: string]: unknown | undefined
   }
-  result: Record<string, unknown>
+  result?: Record<string, unknown>
   start: number
   status: 'failure' | 'interrupted' | 'pending' | 'success'
   tasks?: XoTask[]
@@ -676,7 +715,7 @@ export type XoUser = {
   groups: XoGroup['id'][]
   id: Branded<'user'>
   name?: string
-  permission: string
+  permission: 'none' | 'admin'
   pw_hash?: string
   preferences: Record<string, string>
 }
@@ -850,22 +889,28 @@ export type XapiXoRecord =
   | XoVtpm
   | XoSm
 
-export type NonXapiXoRecord =
+export type NonXapiXoRecord<
+  TActionsByResource extends XoAclSupportedActionsByResource = never,
+  TResource extends string = never,
+> =
   | AnyXoBackupArchive
   | AnyXoJob
   | AnyXoLog
   | XoGroup
   | XoProxy
-  | XoGroup
-  | XoProxy
   | XoJob
   | XoBackupRepository
   | XoSchedule
   | XoServer
   | XoTask
   | XoUser
+  | XoAclRole
+  | XoAclPrivilege<TActionsByResource, TResource>
 
-export type XoRecord = XapiXoRecord | NonXapiXoRecord
+export type XoRecord<
+  TActionsByResource extends XoAclSupportedActionsByResource = never,
+  TResource extends string = never,
+> = XapiXoRecord | NonXapiXoRecord<TActionsByResource, TResource>
 
 export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
 

@xen-orchestra/acl/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/acl/eslint.config.mjs

@@ -0,0 +1,16 @@
+// @ts-check
+
+import eslint from '@eslint/js'
+import tseslint from 'typescript-eslint'
+import { fileURLToPath } from 'node:url'
+import path from 'node:path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
+  languageOptions: {
+    parserOptions: {
+      tsconfigRootDir: __dirname,
+    },
+  }
+})

@xen-orchestra/acl/package.json

@@ -0,0 +1,47 @@
+{
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "main": "dist/index.mjs",
+  "name": "@xen-orchestra/acl",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
+  "version": "1.5.0",
+  "license": "AGPL-3.0-or-later",
+  "private": false,
+  "type": "module",
+  "engines": {
+    "node": ">= 20.18"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "scripts": {
+    "build": "tsc && node scripts/generate-types.mjs",
+    "clean": "rimraf dist/",
+    "dev": "tsc --watch",
+    "postversion": "npm publish --access public",
+    "test": "npm run build && node --test ./dist/**/*.test.mjs",
+    "prebuild": "npm run clean",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.19.0",
+    "@vates/types": "^1.30.1",
+    "rimraf": "^6.0.1",
+    "typescript": "~5.6",
+    "typescript-eslint": "^8.61.0"
+  },
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/acl",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "dependencies": {
+    "complex-matcher": "^1.0.0"
+  }
+}

@xen-orchestra/acl/scripts/generate-types.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+// AUTO-GENERATES dist/generated/privilege-types.d.mts
+// Run via `npm run build` (after tsc).
+
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../dist/actions/index.mjs'
+import { getActionStrings } from '../dist/index.mjs'
+
+function toPascalCase(str) {
+  return str
+    .split(/[-_]/)
+    .map(s => s.charAt(0).toUpperCase() + s.slice(1))
+    .join('')
+}
+
+const resources = Object.keys(SUPPORTED_ACTIONS_BY_RESOURCE)
+
+const typeBlocks = resources.map(resource => {
+  const actionUnion = getActionStrings(resource)
+    .map(action => `'${action}'`)
+    .join(' | ')
+  return `export type ${toPascalCase(resource)}Privilege = {
+  id: XoAclBasePrivilege['id']
+  resource: '${resource}'
+  action: ${actionUnion}
+  selector?: XoAclBasePrivilege['selector']
+  effect: XoAclBasePrivilege['effect']
+  roleId: XoAclBasePrivilege['roleId']
+}`
+})
+
+const anyUnion = resources.map(resource => `  | ${toPascalCase(resource)}Privilege`).join('\n')
+
+const types = `\
+// -----------------------------------------------------------------------
+// AUTO-GENERATED — do not edit manually.
+// Regenerated on every \`npm run build\`.
+// -----------------------------------------------------------------------
+import type { XoAclBasePrivilege } from '@vates/types'
+
+
+${typeBlocks.join('\n\n')}
+
+export type AnyPrivilege =
+${anyUnion}
+`
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..')
+const outDir = join(root, 'dist/generated')
+
+mkdirSync(outDir, { recursive: true })
+writeFileSync(join(outDir, 'privilege-types.d.mts'), types, 'utf-8')

@xen-orchestra/acl/src/actions/acl-privilege.mts

@@ -0,0 +1,11 @@
+export default {
+  create: true,
+  delete: true,
+  read: true,
+  update: {
+    action: true,
+    effect: true,
+    resource: true,
+    selector: true,
+  },
+}

@xen-orchestra/acl/src/actions/acl-role.mts

@@ -0,0 +1,11 @@
+export default {
+  create: true,
+  delete: true,
+  read: true,
+  update: {
+    description: true,
+    groups: true,
+    name: true,
+    users: true,
+  },
+}

@xen-orchestra/acl/src/actions/alarm.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/backup-archive.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/backup-job.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/backup-log.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/backup-repository.mts

@@ -0,0 +1,13 @@
+export default {
+  benchmark: true,
+  create: true,
+  read: true,
+  forget: true,
+  update: {
+    enabled: true,
+    name: true,
+    options: true,
+    proxy: true,
+    url: true,
+  },
+}

@xen-orchestra/acl/src/actions/gpuGroup.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/group.mts

@@ -0,0 +1,9 @@
+export default {
+  create: true,
+  delete: true,
+  read: true,
+  update: {
+    name: true,
+    users: true,
+  },
+}

@xen-orchestra/acl/src/actions/host.mts

@@ -0,0 +1,28 @@
+export default {
+  'allow-vm': true,
+  detach: true,
+  disable: true,
+  enable: true,
+  evacuate: true,
+  export: {
+    logs: true,
+  },
+  forget: true,
+  'join-pool': true,
+  'migrate-receive': true,
+  read: true,
+  reboot: {
+    clean: true,
+    smart: true,
+  },
+  'restart-toolstack': true,
+  'scan-pifs': true,
+  shutdown: {
+    clean: true,
+    emergency: true,
+  },
+  start: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/index.mts

@@ -0,0 +1,80 @@
+import { default as aclPrivilege } from './acl-privilege.mjs'
+import { default as aclRole } from './acl-role.mjs'
+import { default as alarm } from './alarm.mjs'
+import { default as backupArchive } from './backup-archive.mjs'
+import { default as backupJob } from './backup-job.mjs'
+import { default as backupLog } from './backup-log.mjs'
+import { default as backupRepository } from './backup-repository.mjs'
+import { default as gpuGroup } from './gpuGroup.mjs'
+import { default as group } from './group.mjs'
+import { default as host } from './host.mjs'
+import { default as message } from './message.mjs'
+import { default as network } from './network.mjs'
+import { default as pbd } from './pbd.mjs'
+import { default as pci } from './pci.mjs'
+import { default as pgpu } from './pgpu.mjs'
+import { default as pif } from './pif.mjs'
+import { default as pool } from './pool.mjs'
+import { default as proxy } from './proxy.mjs'
+import { default as restoreLog } from './restore-log.mjs'
+import { default as schedule } from './schedule.mjs'
+import { default as server } from './server.mjs'
+import { default as sm } from './sm.mjs'
+import { default as sr } from './sr.mjs'
+import { default as task } from './task.mjs'
+import { default as user } from './user.mjs'
+import { default as vbd } from './vbd.mjs'
+import { default as vdiSnapshot } from './vdi-snapshot.mjs'
+import { default as vdiUnmanaged } from './vdi-unmanaged.mjs'
+import { default as vdi } from './vdi.mjs'
+import { default as vgpu } from './vgpu.mjs'
+import { default as vgpuType } from './vgpuType.mjs'
+import { default as vif } from './vif.mjs'
+import { default as vmController } from './vm-controller.mjs'
+import { default as vmSnapshot } from './vm-snapshot.mjs'
+import { default as vmTemplate } from './vm-template.mjs'
+import { default as vm } from './vm.mjs'
+import { default as vtpm } from './vtpm.mjs'
+
+/**
+ * value without sub action like 'shutdown' mean: 'shutdown:*' (so shutdown mean you can shutdown:clean and shutdown:hard)
+ */
+export const SUPPORTED_ACTIONS_BY_RESOURCE = {
+  'acl-privilege': aclPrivilege,
+  'acl-role': aclRole,
+  alarm,
+  'backup-archive': backupArchive,
+  'backup-job': backupJob,
+  'backup-log': backupLog,
+  'backup-repository': backupRepository,
+  group,
+  gpuGroup,
+  host,
+  message,
+  network,
+  pbd,
+  pci,
+  pgpu,
+  pif,
+  pool,
+  proxy,
+  'restore-log': restoreLog,
+  schedule,
+  server,
+  sm,
+  sr,
+  task,
+  user,
+  vbd,
+  'vdi-snapshot': vdiSnapshot,
+  'vdi-unmanaged': vdiUnmanaged,
+  vdi,
+  vgpu,
+  vgpuType,
+  vif,
+  'vm-controller': vmController,
+  'vm-snapshot': vmSnapshot,
+  'vm-template': vmTemplate,
+  vm,
+  vtpm,
+} as const

@xen-orchestra/acl/src/actions/message.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/network.mts

@@ -0,0 +1,9 @@
+export default {
+  create: true,
+  delete: true,
+  read: true,
+  update: {
+    tags: true,
+    other_config: true,
+  },
+}

@xen-orchestra/acl/src/actions/pbd.mts

@@ -0,0 +1,5 @@
+export default {
+  read: true,
+  plug: true,
+  unplug: true,
+}

@xen-orchestra/acl/src/actions/pci.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/pgpu.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/pif.mts

@@ -0,0 +1,6 @@
+export default {
+  read: true,
+  update: {
+    management: true,
+  },
+}

@xen-orchestra/acl/src/actions/pool.mts

@@ -0,0 +1,14 @@
+export default {
+  'add-host': true,
+  create: {
+    network: true,
+    vm: true,
+  },
+  'emergency-shutdown': true,
+  read: true,
+  'rolling-reboot': true,
+  'rolling-update': true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/proxy.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/restore-log.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/schedule.mts

@@ -0,0 +1,4 @@
+export default {
+  read: true,
+  run: true,
+}

@xen-orchestra/acl/src/actions/server.mts

@@ -0,0 +1,7 @@
+export default {
+  connect: true,
+  create: true,
+  delete: true,
+  disconnect: true,
+  read: true,
+}

@xen-orchestra/acl/src/actions/sm.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/sr.mts

@@ -0,0 +1,16 @@
+export default {
+  create: true,
+  delete: true,
+  forget: true,
+  import: {
+    vdi: true,
+    vm: true,
+  },
+  'migrate-receive': true,
+  read: true,
+  'reclaim-space': true,
+  scan: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/task.mts

@@ -0,0 +1,5 @@
+export default {
+  abort: true,
+  delete: true,
+  read: true,
+}

@xen-orchestra/acl/src/actions/user.mts

@@ -0,0 +1,11 @@
+export default {
+  create: true,
+  delete: true,
+  read: true,
+  update: {
+    name: true,
+    password: true,
+    permission: true,
+    preferences: true,
+  },
+}

@xen-orchestra/acl/src/actions/vbd.mts

@@ -0,0 +1,7 @@
+export default {
+  connect: true,
+  create: true,
+  delete: true,
+  disconnect: true,
+  read: true,
+}

@xen-orchestra/acl/src/actions/vdi-snapshot.mts

@@ -0,0 +1,8 @@
+export default {
+  delete: true,
+  export: true,
+  read: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/vdi-unmanaged.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/vdi.mts

@@ -0,0 +1,15 @@
+export default {
+  boot: true,
+  create: true,
+  delete: true,
+  'export-content': true,
+  'import-content': true,
+  'migrate-send': true,
+  read: true,
+  update: {
+    name_description: true,
+    name_label: true,
+    size: true,
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/vgpu.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/vgpuType.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/actions/vif.mts

@@ -0,0 +1,15 @@
+export default {
+  connect: true,
+  create: true,
+  delete: true,
+  disconnect: true,
+  read: true,
+  update: {
+    allowedIpv4Addresses: true,
+    allowedIpv6Addresses: true,
+    lockingMode: true,
+    rateLimit: true,
+    txChecksumming: true,
+    other_config: true,
+  },
+}

@xen-orchestra/acl/src/actions/vm-controller.mts

@@ -0,0 +1,6 @@
+export default {
+  read: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/vm-snapshot.mts

@@ -0,0 +1,8 @@
+export default {
+  delete: true,
+  export: true,
+  read: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/vm-template.mts

@@ -0,0 +1,9 @@
+export default {
+  delete: true,
+  export: true,
+  instantiate: true,
+  read: true,
+  update: {
+    tags: true,
+  },
+}

@xen-orchestra/acl/src/actions/vm.mts

@@ -0,0 +1,61 @@
+export default {
+  clone: true,
+  delete: true,
+  export: true,
+  'migrate-send': true,
+  pause: true,
+  read: true,
+  reboot: {
+    clean: true,
+    hard: true,
+  },
+  resume: true,
+  'revert-snapshot': true,
+  shutdown: {
+    clean: true,
+    hard: true,
+  },
+  snapshot: true,
+  start: true,
+  suspend: true,
+  unpause: true,
+  update: {
+    affinityHost: true,
+    autoPoweron: true,
+    blockedOperations: true,
+    coresPerSocket: true,
+    cpuCap: true,
+    cpuMask: true,
+    cpuWeight: true,
+    cpus: true,
+    cpusStaticMax: true,
+    creation: true,
+    datasources: true,
+    expNestedHvm: true,
+    hasVendorDevice: true,
+    highAvailability: true,
+    hvmBootFirmware: true,
+    memory: true,
+    memoryMax: true,
+    memoryMin: true,
+    memoryStaticMax: true,
+    nameDescription: true,
+    nameLabel: true,
+    nestedVirt: true,
+    nicType: true,
+    notes: true,
+    PV_args: true,
+    resourceSet: true,
+    secureBoot: true,
+    share: true,
+    startDelay: true,
+    suspendSr: true,
+    tags: true,
+    uefiMode: true,
+    vga: true,
+    videoram: true,
+    viridian: true,
+    virtualizationMode: true,
+    xenStoreData: true,
+  },
+}

@xen-orchestra/acl/src/actions/vtpm.mts

@@ -0,0 +1,3 @@
+export default {
+  read: true,
+}

@xen-orchestra/acl/src/class/privilege.mts

@@ -0,0 +1,119 @@
+import * as CM from 'complex-matcher'
+import type { XoAclPrivilege, XoAclSupportedActions, XoAclSupportedResource } from '@vates/types/lib/xen-orchestra/acl'
+
+import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../actions/index.mjs'
+
+export type SupportedActionsByResource = typeof SUPPORTED_ACTIONS_BY_RESOURCE
+export type SupportedResource = XoAclSupportedResource<SupportedActionsByResource>
+export type SupportedActions<T extends SupportedResource> = XoAclSupportedActions<SupportedActionsByResource, T>
+export type TPrivilege<T extends SupportedResource> = XoAclPrivilege<SupportedActionsByResource, T>
+
+export class Privilege<T extends SupportedResource> {
+  #action: TPrivilege<T>['action']
+  #selector?: (object: object) => boolean
+  #resource: TPrivilege<T>['resource']
+  #effect: TPrivilege<T>['effect']
+
+  constructor({
+    action,
+    selector,
+    resource,
+    effect,
+  }: {
+    action: TPrivilege<T>['action']
+    selector?: TPrivilege<T>['selector']
+    resource: TPrivilege<T>['resource']
+    effect: TPrivilege<T>['effect']
+  }) {
+    Privilege.checkActionIsValid(resource, action)
+
+    this.#action = action
+    this.#selector = selector !== undefined ? CM.parse(selector).createPredicate() : undefined
+    this.#resource = resource
+    this.#effect = effect
+  }
+
+  get effect() {
+    return this.#effect
+  }
+
+  #matchAction<Resource extends SupportedResource = T>(action: SupportedActions<Resource>) {
+    // read:name_label - read:name_label
+    if (this.#action === action) {
+      return true
+    }
+
+    const [namespaceToMatch, actionToMatch] = action.split(':')
+    const [thisNamespace, thisAction] = this.#action.split(':')
+
+    if (thisNamespace === '*') {
+      return true
+    }
+
+    // update:vm - read:name_label
+    if (thisNamespace !== namespaceToMatch) {
+      return false
+    }
+
+    // read - read:name_label
+    if (thisAction === undefined) {
+      return true
+    }
+    // read:name_decription - read:name_label
+    if (thisAction !== actionToMatch) {
+      return false
+    }
+
+    throw new Error(`Unable to verify if ${this.#action} match ${action} `)
+  }
+
+  #matchSelector(object: object) {
+    if (this.#selector === undefined) {
+      return true
+    }
+
+    return this.#selector(object)
+  }
+
+  #matchResource(resource: string): resource is SupportedResource {
+    return resource === this.#resource
+  }
+
+  match<Resource extends SupportedResource = T>(constraint: {
+    action: SupportedActions<Resource>
+    resource: Resource
+    object: object
+  }): boolean {
+    return (
+      this.#matchResource(constraint.resource) &&
+      this.#matchAction(constraint.action) &&
+      this.#matchSelector(constraint.object)
+    )
+  }
+
+  /**
+   * Throw if action not supported
+   */
+  static checkActionIsValid<T extends SupportedResource>(resource: T, action: SupportedActions<T>) {
+    const supportedActions = SUPPORTED_ACTIONS_BY_RESOURCE[resource]
+    if (supportedActions === undefined) {
+      throw new Error(`${resource} resource not supported`)
+    }
+
+    if (action === '*') {
+      return
+    }
+
+    const segments = action.split(':')
+    let _action: object | undefined = undefined
+    segments.forEach(segment => {
+      _action = (_action ?? supportedActions)[segment]
+
+      if (_action === undefined) {
+        throw new Error(
+          `${action} action not supported for the resource: ${resource}. See ${JSON.stringify(supportedActions)}`
+        )
+      }
+    })
+  }
+}

@xen-orchestra/acl/src/generated/privilege-types.mts

@@ -0,0 +1,10 @@
+// DO NOT EDIT
+// Overwritten by scripts/generate-types.mjs after running npm run build
+// scripts/generate-types.mjs will replace this file to automatically generate and export all privileged types.
+// This makes external use easier because it's not necessary to resolve the type every time.
+
+import type { Privilege, SupportedResource } from '../index.mjs'
+
+export type AnyPrivilege = {
+  [Resource in SupportedResource]: Privilege<Resource>
+}[SupportedResource]

@xen-orchestra/acl/src/index.mts

@@ -0,0 +1,167 @@
+import assert from 'node:assert'
+import type { XoUser } from '@vates/types/xo'
+
+import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
+import {
+  Privilege as CPrivilege,
+  TPrivilege as Privilege,
+  SupportedActions,
+  SupportedActionsByResource,
+  SupportedResource,
+} from './class/privilege.mjs'
+
+// Export all types constructed using SUPPORTED_ACTIONS_BY_RESOURCE
+export type { Privilege, SupportedActions, SupportedActionsByResource, SupportedResource }
+export { SUPPORTED_ACTIONS_BY_RESOURCE }
+
+export type * from './generated/privilege-types.mjs'
+
+import { AnyPrivilege } from './generated/privilege-types.mjs'
+
+export type AnyPrivilegeOnParam = {
+  [Resource in SupportedResource]: {
+    user: XoUser
+    resource: Resource
+    action: SupportedActions<Resource>
+    objects: object | object[]
+  }
+}[SupportedResource]
+
+export function hasPrivilegeOn<T extends SupportedResource>({
+  user,
+  action,
+  resource,
+  objects,
+  userPrivileges,
+}: {
+  user: XoUser
+  resource: T
+  action: SupportedActions<T>
+  objects: object | object[]
+  userPrivileges: AnyPrivilege[]
+}) {
+  // Function that will be called outside of the module
+  // We cannot be sure types are respected
+  assert.strictEqual(typeof user?.permission, 'string')
+  assert.strictEqual(typeof action, 'string')
+  assert.strictEqual(typeof resource, 'string')
+  assert.strictEqual(objects === undefined, false)
+
+  CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
+
+  if (user.permission === 'admin') {
+    return true
+  }
+
+  const effectsByObject = new Map<object, Privilege<T>['effect'][]>()
+  if (Array.isArray(objects)) {
+    objects.forEach(obj => effectsByObject.set(obj, []))
+  } else {
+    effectsByObject.set(objects, [])
+  }
+
+  for (const userPrivilege of userPrivileges) {
+    const privilege = new CPrivilege(userPrivilege as Privilege<typeof userPrivilege.resource>)
+
+    for (const [object, effects] of effectsByObject.entries()) {
+      if (!privilege.match({ action, resource, object })) {
+        continue
+      }
+      effects.push(privilege.effect)
+    }
+  }
+
+  if (
+    Array.from(effectsByObject.values()).some(
+      effects => effects.length === 0 || effects.some(effect => effect === 'deny')
+    )
+  ) {
+    return false
+  }
+  return true
+}
+
+export function getMissingPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
+  return params
+    .filter(
+      param =>
+        !hasPrivilegeOn({
+          user: param.user,
+          resource: param.resource,
+          action: param.action as SupportedActions<typeof param.resource>,
+          objects: param.objects,
+          userPrivileges,
+        })
+    )
+    .map(({ action, objects, resource }) => {
+      let objectIds: unknown[] | undefined
+      let objectId: unknown | undefined
+
+      if (Array.isArray(objects)) {
+        if (objects.length > 1) {
+          objectIds = []
+          objects.forEach(obj => {
+            if ('id' in obj) {
+              objectIds!.push(obj.id)
+            }
+          })
+        } else {
+          objects = objects?.[0] ?? {}
+        }
+      }
+      if (!Array.isArray(objects)) {
+        objectId = 'id' in objects ? objects.id : undefined
+      }
+
+      return {
+        objectId,
+        objectIds,
+        action,
+        resource,
+      }
+    })
+}
+
+export function hasPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
+  return getMissingPrivileges(params, userPrivileges).length === 0
+}
+
+export function filterObjectsWithPrivilege<Resource extends SupportedResource, Object extends object>(param: {
+  user: XoUser
+  resource: Resource
+  action: SupportedActions<Resource>
+  objects: Object[]
+  userPrivileges: AnyPrivilege[]
+}) {
+  return param.objects.filter(obj => hasPrivilegeOn({ ...param, objects: obj }))
+}
+
+// Utils
+
+/**
+ * Mirror GetKeysRecursively from @vates/types
+ */
+export function getActionStrings<Resource extends SupportedResource, Actions = SupportedActions<Resource>>(
+  resource: Resource
+): Actions[] {
+  type NestedActions = { [key: string]: boolean | NestedActions }
+
+  function collectActions(obj: NestedActions, prefix?: Actions): Actions[] {
+    const actions: Actions[] = ['*' as Actions]
+
+    for (const [key, value] of Object.entries(obj)) {
+      const action = (prefix !== undefined ? `${prefix}:${key}` : key) as Actions
+      actions.push(action)
+
+      if (typeof value === 'boolean') {
+        continue
+      }
+
+      actions.push(...collectActions(value, action))
+    }
+
+    return actions
+  }
+
+  return collectActions(SUPPORTED_ACTIONS_BY_RESOURCE[resource])
+}

@xen-orchestra/acl/src/tests/acl.test.mts

@@ -0,0 +1,378 @@
+import { test, suite } from 'node:test'
+import assert from 'node:assert'
+import type { Branded } from '@vates/types/common'
+import type { XoUser } from '@vates/types/xo'
+
+import { Privilege } from '../class/privilege.mjs'
+import type { Privilege as TPrivilege } from '../index.mjs'
+import {
+  filterObjectsWithPrivilege,
+  getActionStrings,
+  getMissingPrivileges,
+  hasPrivilegeOn,
+  hasPrivileges,
+} from '../index.mjs'
+
+suite('Privilege.checkActionIsValid behavior', () => {
+  test('throw on invalid resource', () => {
+    // @ts-expect-error test invalid action
+    assert.throws(() => Privilege.checkActionIsValid('foo', 'read'))
+  })
+
+  test('throw on invalid action', () => {
+    // @ts-expect-error test invalid action
+    assert.throws(() => Privilege.checkActionIsValid('vm', 'foo:vm'))
+  })
+
+  test('should work', () => {
+    assert.doesNotThrow(() => Privilege.checkActionIsValid('vm', 'read'))
+  })
+})
+
+suite('Privilege.match behavior', () => {
+  const privilege = new Privilege({ action: 'read', resource: 'vm', effect: 'allow' })
+  const object = { id: 1, name_label: 'foo' }
+
+  test('Should not match because of the action', () => {
+    assert.strictEqual(privilege.match({ resource: 'vm', action: 'start', object }), false)
+  })
+
+  test('Should not match because of the resource', () => {
+    assert.strictEqual(privilege.match({ resource: 'host', action: 'read', object }), false)
+  })
+
+  test('Should match', () => {
+    assert.strictEqual(privilege.match({ resource: 'vm', action: 'read', object }), true)
+  })
+
+  test('Should match even if the action is not equal to the action privilege', () => {
+    const shutdownPrivilege = new Privilege({ action: 'shutdown', resource: 'vm', effect: 'allow' })
+    assert.strictEqual(shutdownPrivilege.match({ resource: 'vm', action: 'shutdown:clean', object }), true)
+  })
+
+  test('Should match because the action is *', () => {
+    const vmPrivilege = new Privilege({ action: '*', resource: 'vm', effect: 'allow' })
+    assert.strictEqual(vmPrivilege.match({ action: 'reboot:hard', resource: 'vm', object: object }), true)
+  })
+
+  test('Should not match due to insufficient privilege', () => {
+    const cleanShutdownPrivilege = new Privilege({ action: 'shutdown:clean', resource: 'vm', effect: 'allow' })
+    assert.strictEqual(cleanShutdownPrivilege.match({ resource: 'vm', action: 'shutdown', object }), false)
+  })
+})
+
+suite('ACL V2 behavior', async () => {
+  // === Setup ===
+  const admin = {
+    id: '1' as XoUser['id'],
+    groups: [],
+    permission: 'admin',
+    email: 'admin@admin.net',
+    preferences: {},
+  } satisfies XoUser
+
+  const user = {
+    id: '2' as XoUser['id'],
+    groups: [],
+    permission: 'none',
+    email: 'user@user.net',
+    preferences: {},
+  } satisfies XoUser
+
+  const privilegeId = 'fake-priv-test-id' as Branded<'acl-v2-privilege'>
+  const roleId = 'fake-role-test-id' as Branded<'acl-v2-role'>
+
+  // === Objects
+  const poolWithoutPrivilege = { id: 'pool-id' }
+  const alpineVm = { id: 'alpine', creation: { creator: admin.id } }
+  const almaVm = { id: 'alma', creation: { creator: user.id } }
+  const xoaVm = { id: 'xoa' }
+  const template = { id: 'template-id', $pool: '1' }
+  const templateDeny = { id: 'template-deny-id', $pool: '2' }
+  const vdis = [{ $SR: '2' }, { $SR: '2' }]
+  const vdisMultipleSrs = [{ $SR: '1' }, { $SR: '2' }]
+  const vifs = [{ $network: '1' }]
+  const iso = { id: 'iso-vdi-id', $SR: '3' }
+  const hostAffinity = { id: 'affinity-host-id', $pool: '1' }
+
+  // === DB Privileges
+  const vmRead = { id: privilegeId, action: 'read', resource: 'vm', effect: 'allow', roleId } satisfies TPrivilege<'vm'>
+  const vmCleanShutdown = {
+    id: privilegeId,
+    action: 'shutdown:clean',
+    resource: 'vm',
+    selector: `creation:creator:${user.id}`,
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'vm'>
+  const xoaVmDenyRead = {
+    id: privilegeId,
+    action: 'read',
+    resource: 'vm',
+    effect: 'deny',
+    selector: `id: ${xoaVm.id}`,
+    roleId,
+  } satisfies TPrivilege<'vm'>
+  const vdiCreate = {
+    id: privilegeId,
+    action: 'create',
+    resource: 'vdi',
+    selector: '$SR:2',
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'vdi'>
+  const vifCreate = {
+    id: privilegeId,
+    action: 'create',
+    resource: 'vif',
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'vif'>
+  const templateInstantiate = {
+    id: privilegeId,
+    action: 'instantiate',
+    resource: 'vm-template',
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'vm-template'>
+  const templateInstantiateDeny = {
+    id: privilegeId,
+    action: 'instantiate',
+    resource: 'vm-template',
+    selector: '$pool:2',
+    effect: 'deny',
+    roleId,
+  } satisfies TPrivilege<'vm-template'>
+  const isoVdiUse = {
+    id: privilegeId,
+    action: 'boot',
+    resource: 'vdi',
+    selector: '$SR:3',
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'vdi'>
+  const hostVm = {
+    id: privilegeId,
+    action: 'allow-vm',
+    resource: 'host',
+    effect: 'allow',
+    roleId,
+  } satisfies TPrivilege<'host'>
+
+  const allPrivileges = [
+    vmRead,
+    vmCleanShutdown,
+    xoaVmDenyRead,
+    vdiCreate,
+    vifCreate,
+    templateInstantiate,
+    templateInstantiateDeny,
+    isoVdiUse,
+    hostVm,
+  ]
+
+  // === Test ===
+  suite('hasPrivilegeOn behavior', () => {
+    test('Should always return true for admin', () => {
+      assert.strictEqual(
+        hasPrivilegeOn<'pool'>({
+          user: admin,
+          action: 'read',
+          resource: 'pool',
+          objects: poolWithoutPrivilege,
+          userPrivileges: allPrivileges,
+        }),
+        true
+      )
+    })
+
+    test('Should return false by default for non admin', () => {
+      assert.strictEqual(
+        hasPrivilegeOn({
+          user,
+          action: 'read',
+          resource: 'pool',
+          objects: poolWithoutPrivilege,
+          userPrivileges: allPrivileges,
+        }),
+        false
+      )
+    })
+
+    test('Should return true if user has privilege', () => {
+      assert.strictEqual(
+        hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: alpineVm, userPrivileges: allPrivileges }),
+        true
+      )
+    })
+
+    test('Should return false if user have a both deny and allow privileges', () => {
+      assert.strictEqual(
+        hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: xoaVm, userPrivileges: allPrivileges }),
+        false
+      )
+    })
+
+    test('Should handle multiple and single object', () => {
+      assert.strictEqual(
+        hasPrivilegeOn({
+          user,
+          action: 'read',
+          resource: 'vm',
+          objects: [alpineVm, almaVm],
+          userPrivileges: allPrivileges,
+        }),
+        true
+      )
+      assert.strictEqual(
+        hasPrivilegeOn({
+          user,
+          action: 'read',
+          resource: 'vm',
+          objects: [alpineVm, almaVm, xoaVm],
+          userPrivileges: allPrivileges,
+        }),
+        false
+      )
+    })
+
+    test('Should allow clean shutdown only own VMs', () => {
+      assert.strictEqual(
+        hasPrivilegeOn({
+          user,
+          action: 'shutdown:clean',
+          resource: 'vm',
+          objects: almaVm,
+          userPrivileges: allPrivileges,
+        }),
+        true
+      )
+      assert.strictEqual(
+        hasPrivilegeOn({
+          user,
+          action: 'shutdown:clean',
+          resource: 'vm',
+          objects: [almaVm, alpineVm],
+          userPrivileges: allPrivileges,
+        }),
+        false
+      )
+    })
+  })
+
+  suite('getMissingPrivileges behavior', () => {
+    test('Should return empty array if no missing privileges', () => {
+      const missingPrivileges = getMissingPrivileges(
+        [
+          { user, action: 'shutdown:clean', resource: 'vm', objects: almaVm },
+          { user, action: 'instantiate', resource: 'vm-template', objects: template },
+          { user, action: 'create', resource: 'vdi', objects: vdis },
+          { user, action: 'create', resource: 'vif', objects: vifs },
+          { user, action: 'boot', resource: 'vdi', objects: iso },
+          { user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
+        ],
+        allPrivileges
+      )
+
+      assert.strictEqual(missingPrivileges.length, 0)
+    })
+
+    test('Should return only missing privileges', () => {
+      const missingPrivileges = getMissingPrivileges(
+        [
+          { user, action: 'instantiate', resource: 'vm-template', objects: template },
+          { user, action: 'create', resource: 'vdi', objects: vdis },
+          { user, action: 'create', resource: 'vif', objects: vifs },
+          { user, action: 'boot', resource: 'vdi', objects: iso },
+          { user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
+          { user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege },
+          { user, action: '*', resource: 'pool', objects: poolWithoutPrivilege },
+        ],
+        allPrivileges
+      )
+
+      assert.strictEqual(missingPrivileges.length, 2)
+    })
+
+    test('Should return objectId if only one object in the missing privilege', () => {
+      const missingPrivileges = getMissingPrivileges(
+        [{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege }],
+        allPrivileges
+      )
+
+      assert.strictEqual(missingPrivileges.length, 1)
+      assert.notStrictEqual(missingPrivileges[0]!.objectId, undefined)
+    })
+
+    test('Should return objectIds if multiple objects in the missing privilege', () => {
+      const missingPrivileges = getMissingPrivileges(
+        [{ user, action: 'read', resource: 'pool', objects: [poolWithoutPrivilege, poolWithoutPrivilege] }],
+        allPrivileges
+      )
+
+      assert.strictEqual(missingPrivileges.length, 1)
+      assert.notStrictEqual(missingPrivileges[0]!.objectIds, undefined)
+    })
+  })
+
+  suite('hasPrivileges behavior', () => {
+    test('Should return false if at least one privilege deny', () => {
+      const result = hasPrivileges(
+        [
+          { user, action: 'instantiate', resource: 'vm-template', objects: templateDeny },
+          { user, action: 'create', resource: 'vdi', objects: vdis },
+          { user, action: 'create', resource: 'vif', objects: vifs },
+          { user, action: 'boot', resource: 'vdi', objects: iso },
+          { user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
+        ],
+        allPrivileges
+      )
+      assert.strictEqual(result, false)
+    })
+    test("Should return false if at least one privilege didn't match", () => {
+      const result = hasPrivileges(
+        [
+          { user, action: 'instantiate', resource: 'vm-template', objects: template },
+          { user, action: 'create', resource: 'vdi', objects: vdisMultipleSrs },
+          { user, action: 'create', resource: 'vif', objects: vifs },
+          { user, action: 'boot', resource: 'vdi', objects: iso },
+          { user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
+        ],
+        allPrivileges
+      )
+      assert.strictEqual(result, false)
+    })
+  })
+
+  suite('filterObjectsWithPrivilege behavior', () => {
+    test('Should return only objects that match privileges', () => {
+      let vms = [almaVm, alpineVm, xoaVm, xoaVm]
+      assert.strictEqual(vms.length, 4)
+      vms = filterObjectsWithPrivilege({
+        user,
+        action: 'read',
+        resource: 'vm',
+        objects: vms,
+        userPrivileges: allPrivileges,
+      })
+      assert.strictEqual(vms.length, 2)
+    })
+  })
+})
+
+suite('getActionStrings', () => {
+  test('returns flat actions for a resource without nesting', () => {
+    const actions = getActionStrings('host')
+    assert.ok(actions.includes('read'))
+    // @ts-ignore A TypeScript error is thrown because `foo` is not of type `SupportedActions<'vm'>`.
+    // To test for a negative value, the error is ignored.
+    assert.strictEqual(actions.includes('foo'), false)
+  })
+
+  test('parent action is included alongside its children', () => {
+    const actions = getActionStrings('vm')
+    assert.ok(actions.includes('shutdown'))
+    assert.ok(actions.includes('shutdown:clean'))
+    assert.ok(actions.includes('shutdown:hard'))
+  })
+})

@xen-orchestra/acl/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+
+    "strict": true,
+    "module": "nodenext",
+    "lib": ["ESNext"],
+    "moduleResolution": "nodenext",
+    "target": "ES2022",
+    "skipLibCheck": true,
+    "types": ["node"], // Only use types from Node.js and packages that are explicitly imported
+    "noImplicitAny": false, // otherwise it will throw an error when importing packages without type definition
+    "noUncheckedIndexedAccess": true,
+    "declaration": true
+  },
+  "include": ["./src/**/*"],
+  "exclude": ["./node_modules", "./dist"]
+}

@xen-orchestra/backup-archive/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/backup-archive/eslint.config.mjs

@@ -0,0 +1,21 @@
+// @ts-check
+
+import eslint from '@eslint/js'
+import tseslint from 'typescript-eslint'
+import { fileURLToPath } from 'node:url'
+import path from 'node:path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
+  languageOptions: {
+    parserOptions: {
+      tsconfigRootDir: __dirname,
+    },
+  },
+  ignores: ['dist'],
+  rules: {
+    // Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
+    '@typescript-eslint/no-empty-object-type': 'off',
+  },
+})

@xen-orchestra/backup-archive/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@xen-orchestra/backup-archive",
+  "version": "1.1.1",
+  "description": "Helper to handle generic backup archives",
+  "license": "ISC",
+  "author": "",
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./disks": "./dist/disks/index.mjs"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.19.0",
+    "rimraf": "^6.0.1",
+    "@types/node": "^22.3.0",
+    "typescript": "~5.6",
+    "typescript-eslint": "^8.61.0",
+    "fs-extra": "^11.1.0",
+    "sinon": "^18.0.0",
+    "tar-stream": "^3.1.8",
+    "@types/tar-stream": "^3.1.4",
+    "tmp": "^0.2.1",
+    "@vates/types": "^1.30.1"
+  },
+  "dependencies": {
+    "@xen-orchestra/log": "^0.7.1",
+    "@xen-orchestra/fs": "^4.9.3",
+    "@xen-orchestra/disk-transform": "^1.3.3",
+    "@vates/task": "0.7.0",
+    "golike-defer": "0.5.1",
+    "uuid": "^9.0.0",
+    "promise-toolbox": "^0.21.0",
+    "@vates/async-each": "^1.0.1",
+    "vhd-lib": "^4.16.1"
+  },
+  "scripts": {
+    "build": "yarn run clean && tsc",
+    "dev": "tsc --watch",
+    "postversion": "npm publish --access public",
+    "test": "yarn run clean && tsc && node --test **/*.test.mjs",
+    "test-integration": "yarn run clean && tsc && node --test **/*.integ.mjs",
+    "clean": "rimraf dist/"
+  },
+  "private": false,
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/backup-archive",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "engines": {
+    "node": ">=20.17"
+  }
+}

@xen-orchestra/backup-archive/src/RemoteDiskLineage.mts

@@ -0,0 +1,438 @@
+import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
+import {
+  MergeRemoteDisk,
+  openDisk,
+  openDisposableDisk,
+  RemoteDisk,
+  isDisk,
+  openDiskChain,
+} from '@xen-orchestra/backup-archive/disks'
+import Disposable from 'promise-toolbox/Disposable'
+import {
+  CleanResult,
+  DEFAULT_MERGE_CONCURRENCY,
+  DEFAULT_REMOVE_CONCURRENCY,
+  ResolvedBackupCleanOptions,
+} from './VmBackup.types.mjs'
+import { asyncEach } from '@vates/async-each'
+import { Task } from '@vates/task'
+import { limitConcurrency } from 'limit-concurrency-decorator'
+import assert from 'node:assert/strict'
+
+const defaultMergeLimiter = limitConcurrency(1)
+const MERGE_STATE_FILE_RE = /^\.(.+)\.merge\.json$/
+/**
+ * Tracks the disk chain for a single VDI across all backup snapshots.
+ * Owns merge and deletion decisions for its chain given which disks are still
+ * referenced by active backups (accumulated via addActiveDiskPath).
+ */
+export class RemoteDiskLineage {
+  #handler: RemoteHandlerAbstract
+  #vdiDir: string
+  #opts: ResolvedBackupCleanOptions
+
+  // Concrete disk paths discovered in this VDI directory (normalized)
+  #diskPaths: Set<string> = new Set()
+  // Disk paths that exist on disk but failed to open (corrupted)
+  #brokenDiskPaths: Set<string> = new Set()
+  // child path: parent path (from disk header)
+  #parentOf: Map<string, string> = new Map()
+  // parent path: child path
+  #childOf: Map<string, string> = new Map()
+  // Disk paths declared active by their owning archives (accumulated across all referencing archives)
+  #activeDiskPaths: Set<string> = new Set()
+  // Interrupted merges: normalized parent path { stateFilePath, chain }
+  #interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
+
+  constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
+    this.#handler = handler
+    this.#vdiDir = vdiDir
+    this.#opts = opts
+  }
+
+  #unregisterDisk(diskPath: string): void {
+    this.#diskPaths.delete(diskPath)
+    const parent = this.#parentOf.get(diskPath)
+    if (parent !== undefined) {
+      this.#parentOf.delete(diskPath)
+      this.#childOf.delete(parent)
+    }
+    const child = this.#childOf.get(diskPath)
+    if (child !== undefined) {
+      this.#childOf.delete(diskPath)
+      this.#parentOf.delete(child)
+    }
+  }
+
+  /**
+   * Lists disk files and reads their headers to build the parent-child chain.
+   * Also detects interrupted merge state files.
+   */
+  async init(): Promise<void> {
+    let files: string[]
+    try {
+      files = await this.#handler.list(this.#vdiDir, { prependDir: true })
+    } catch (error) {
+      if (error?.code === 'NOT_SUPPORTED') throw error
+      this.#opts.logWarn('failed to list VDI directory', { vdiDir: this.#vdiDir, error })
+      return
+    }
+
+    for (const filePath of files) {
+      if (isDisk(filePath)) {
+        this.#diskPaths.add(normalize(filePath))
+      }
+    }
+
+    const mergeParentPaths = new Set<string>()
+    for (const filePath of files) {
+      const match = MERGE_STATE_FILE_RE.exec(basename(filePath))
+      if (match !== null) {
+        mergeParentPaths.add(normalize(this.#vdiDir + '/' + match[1]))
+      }
+    }
+
+    const uuidToPath = new Map<string, string>()
+    for (const diskPath of this.#diskPaths) {
+      let disk: RemoteDisk | undefined
+      try {
+        if (mergeParentPaths.has(diskPath)) {
+          disk = await openDisk({ handler: this.#handler, path: diskPath, force: true })
+        } else {
+          disk = await openDisk({ handler: this.#handler, path: diskPath })
+        }
+      } catch (error) {
+        if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
+          throw error
+        }
+        this.#opts.logWarn('failed to open disk', { path: diskPath, error })
+        // Broken disks will be removed by the clean phase
+        // because they are marked as NOT active
+        this.#brokenDiskPaths.add(diskPath)
+        continue
+      }
+      try {
+        const uuid = disk.getUuid()
+        // Detect Disks with the same UUIDs
+        // Due to a bug introduced in a1bcd35e2
+        const existingPath = uuidToPath.get(uuid)
+        if (existingPath !== undefined) {
+          this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
+          let existingDisk: RemoteDisk | undefined
+          try {
+            existingDisk = await openDisk({ handler: this.#handler, path: existingPath })
+            if (existingDisk.containsAllDataOf(disk)) {
+              this.#opts.logWarn('should drop duplicate disk, existing is superset', { dropped: diskPath })
+              continue
+            } else if (disk.containsAllDataOf(existingDisk)) {
+              this.#opts.logWarn('should drop old duplicate disk, new is superset', { dropped: existingPath })
+            } else {
+              this.#opts.logWarn('duplicate disks have different content, should first-seen', { dropped: diskPath })
+              continue
+            }
+          } catch (error) {
+            this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
+            this.#unregisterDisk(existingPath)
+          } finally {
+            await existingDisk?.close()
+          }
+        }
+        uuidToPath.set(uuid, diskPath)
+
+        if (disk.isDifferencing()) {
+          const parentPath = disk.getParentPath()
+          this.#parentOf.set(diskPath, parentPath)
+          if (this.#childOf.has(parentPath)) {
+            const error = new Error('this script does not support multiple children', {
+              cause: {
+                parent: parentPath,
+                child1: this.#childOf.get(parentPath),
+                child2: diskPath,
+              },
+            })
+            throw error
+          }
+          this.#childOf.set(parentPath, diskPath)
+        }
+      } catch (error) {
+        this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
+      } finally {
+        await disk.close()
+      }
+    }
+
+    this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
+      this.#handler,
+      this.#vdiDir,
+      files,
+      uuidToPath,
+      this.#childOf
+    )
+
+    const propagateBroken = (diskPath: string) => {
+      if (!this.#brokenDiskPaths.has(diskPath)) {
+        this.#brokenDiskPaths.add(diskPath)
+        const child = this.#childOf.get(diskPath)
+        if (child !== undefined) {
+          propagateBroken(child)
+        }
+      }
+    }
+
+    for (const [child, parent] of this.#parentOf) {
+      if (!this.#diskPaths.has(parent) || this.#brokenDiskPaths.has(parent)) {
+        this.#opts.logWarn('disk broken: parent missing or broken', { child, parent })
+        propagateBroken(child)
+      }
+    }
+  }
+
+  /**
+   * Returns true if the disk path exists on disk and opened without errors during init.
+   * Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
+   */
+  isPathAccessible(diskPath: string): boolean {
+    return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
+  }
+
+  /** Returns true if the disk exists but failed to open during init. */
+  isBroken(diskPath: string): boolean {
+    return this.#brokenDiskPaths.has(diskPath)
+  }
+
+  /** Verifies that every parent referenced in disk headers exists in the directory. */
+  async check(): Promise<void> {
+    for (const [child, parent] of this.#parentOf) {
+      if (!this.#diskPaths.has(parent)) {
+        this.#opts.logWarn('parent disk is missing', { child, parent })
+      }
+    }
+  }
+
+  /**
+   * Declares disk paths as active for this lineage.
+   * Called by each archive that references this lineage after it determines it is complete.
+   * Accumulated across all referencing archives before clean() is called.
+   */
+  addActiveDiskPath(diskPath: string): void {
+    this.#activeDiskPaths.add(normalize(diskPath))
+  }
+
+  /**
+   * For each orphan disk:
+   * - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
+   * - If it has no active descendant → delete it (if opts.remove).
+   *
+   * Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
+   * Deletes orphan merge state files (for missing disks) when opts.remove is true.
+   *
+   * Requires activeDiskPaths to be completed to have been called by all referencing archives before this.
+   * @returns Set of deleted disk paths.
+   */
+  async clean({
+    remove = this.#opts.remove ?? false,
+    merge = this.#opts.merge ?? false,
+  }: { remove?: boolean; merge?: boolean } = {}): Promise<CleanResult> {
+    const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
+    const toDelete = new Set<string>()
+    const toMerge: { chain: string[]; isResuming: boolean }[] = []
+    const visited = new Set<string>()
+
+    // Walk from an orphan toward its descendants.
+    // Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
+    // or undefined if the subtree is entirely orphaned (mark for deletion).
+    const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
+      if (visited.has(diskPath)) {
+        return undefined
+      }
+      visited.add(diskPath)
+
+      if (!orphanDisks.has(diskPath)) {
+        // Active disk, anchor of the merge chain
+        return [diskPath]
+      }
+
+      const child = this.#childOf.get(diskPath)
+      if (child !== undefined) {
+        const chain = getUsedChildChainOrDelete(child)
+        if (chain !== undefined) {
+          chain.unshift(diskPath)
+          return chain
+        }
+      }
+
+      toDelete.add(diskPath)
+      return undefined
+    }
+
+    // Process interrupted merges first so their disks are protected from the orphan loop
+    for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
+      if (!this.#diskPaths.has(parentPath)) {
+        this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
+        if (remove) {
+          this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
+          await this.#handler.unlink(stateFilePath)
+        }
+        continue
+      }
+
+      let chain: string[] | undefined
+      if (stateChain !== undefined) {
+        const existing = stateChain.filter(p => this.#diskPaths.has(p))
+        if (existing.length >= 2) chain = existing
+      }
+
+      if (chain !== undefined) {
+        chain.forEach(p => visited.add(p))
+        toMerge.push({ chain, isResuming: true })
+      }
+    }
+
+    for (const orphan of orphanDisks) {
+      if (!visited.has(orphan)) {
+        const parent = this.#parentOf.get(orphan)
+        if (parent === undefined || !orphanDisks.has(parent)) {
+          const chain = getUsedChildChainOrDelete(orphan)
+          if (chain !== undefined && chain.length > 1) {
+            toMerge.push({ chain, isResuming: false })
+          }
+        }
+      }
+    }
+
+    if (remove) {
+      await asyncEach(
+        Array.from(toDelete),
+        async path => {
+          this.#opts.logInfo('deleting unused disk', { path })
+          try {
+            const disk = await openDisk({ handler: this.#handler, path, ignoreBlockIndexes: true })
+            await disk.unlink({ force: true })
+            // Remove from tracking so #cleanOrphanDataFiles skips this now-deleted path
+            this.#unregisterDisk(path)
+          } catch (error) {
+            this.#opts.logWarn('failed to delete unused disk', { path, error })
+          }
+        },
+        { concurrency: DEFAULT_REMOVE_CONCURRENCY }
+      )
+    }
+
+    if (toMerge.length > 0) {
+      this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
+    }
+
+    // mergeTargetPath: final size of the disk everything was merged into
+    const mergedSizes = new Map<string, number>()
+    if (merge) {
+      const limitedMergeChain = defaultMergeLimiter(this.#mergeChain.bind(this))
+      const doMerge = async () => {
+        await asyncEach(
+          toMerge,
+          async ({ chain, isResuming }) => {
+            const { finalDiskSize, mergeTargetPath } = await limitedMergeChain([...chain], isResuming)
+            mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
+            // parentPath alias deleted by parentDisk.rename(mergeTargetPath)
+            // intermediates deleted by childDisk.unlink() when removeUnused=true
+            // Unregister so #cleanOrphanDataFiles does not read already-deleted aliases
+            // and produce false "missing target of alias" warnings.
+            for (const deletedPath of chain.slice(0, -1)) {
+              this.#unregisterDisk(deletedPath)
+            }
+          },
+          { concurrency: this.#opts.mergeConcurrency ?? DEFAULT_MERGE_CONCURRENCY }
+        )
+      }
+      if (toMerge.length !== 0) {
+        await Task.run({ properties: { name: 'merge' } }, doMerge)
+      }
+    }
+
+    await this.#cleanOrphanDataFiles(remove)
+
+    return {
+      removedFiles: Array.from(toDelete),
+      merge: toMerge.length > 0,
+      size: [...mergedSizes.values()].reduce((total, subSize) => total + subSize, 0),
+      mergedSizes,
+    }
+  }
+
+  /**
+   * Merges ancestors into the active child at the end of the chain.
+   * Returns the final size of the merge target and its path.
+   */
+  async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
+    assert.ok(chain.length >= 2, `look to merge a chain shorter than 2 ${JSON.stringify(chain)}`)
+    const parentPath = chain[0]
+    this.#opts.logInfo('merging disk chain', { chain: [...chain] }) // need a copy here to log full chain
+    // The last disk in the chain is the active one that everything gets merged into
+    const mergeTargetPath = chain.pop()!
+
+    const merger = new MergeRemoteDisk(this.#handler, {
+      logInfo: this.#opts.logInfo,
+      removeUnused: this.#opts.remove,
+      mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
+      onProgress: this.#opts.onProgress,
+    })
+    let parentDisk: RemoteDisk | undefined
+    let childDisk: RemoteDisk | undefined
+    try {
+      parentDisk = await openDisk({ handler: this.#handler, path: parentPath, force: isResuming })
+      childDisk = await openDiskChain({
+        handler: this.#handler,
+        path: mergeTargetPath,
+        until: parentPath,
+        force: isResuming,
+      })
+
+      const { finalDiskSize } = (await merger.merge(parentDisk, childDisk!))!
+      return { finalDiskSize, mergeTargetPath }
+    } finally {
+      await parentDisk?.close().catch(error => {
+        this.#opts.logWarn('Error closing parent Disk', error)
+      })
+      await childDisk?.close().catch(error => {
+        this.#opts.logWarn('Error closing child Disk', error)
+      })
+    }
+  }
+
+  /**
+   * Validates each disk integrity, then scans discovered data subdirs
+   * and deletes any files not associated to a known disk.
+   */
+  async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
+    const claimedFiles = new Set<string>()
+    for (const diskPath of this.#diskPaths) {
+      const claimed = await Disposable.use(
+        openDisposableDisk({ handler: this.#handler, path: diskPath, ignoreBlockIndexes: true }),
+        async disk => {
+          await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
+          return disk.listAssociatedFiles(this.#vdiDir)
+        }
+      )
+      for (const f of claimed) claimedFiles.add(normalize(f))
+    }
+
+    const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
+    for (const dataDir of dataDirs) {
+      const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
+      for (const item of items) {
+        if (!claimedFiles.has(normalize(item))) {
+          this.#opts.logWarn('orphaned data file', { path: item })
+          if (remove) {
+            await this.#handler.unlink(item).catch(async err => {
+              if (err?.code === 'EISDIR') {
+                await this.#handler.rmtree(item)
+              } else {
+                this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
+              }
+            })
+          }
+        }
+      }
+    }
+  }
+}

@xen-orchestra/backup-archive/src/VmBackup.types.mts

@@ -0,0 +1,74 @@
+import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+
+export interface PartialBackupMetadata {
+  mode: 'full' | 'delta'
+  xva?: string
+  vhds?: Record<string, string>
+  vdis?: Record<string, unknown>
+  isVhdDifferencing?: Record<string, boolean>
+  size?: number
+  jobId: string
+  scheduleId: string
+  timestamp: number
+  // 'true' when written by code including the non-NBD qcow2 corruption fix and its force-full
+  // gate; its absence marks a potentially-corrupt older backup and makes the gate re-probe.
+  // Preserved (never stamped) across merges here.
+  // @todo remove this code and the logic associated on 2027 01 01
+  includeNonNbdQcow2Fix?: boolean
+}
+export const DEFAULT_MERGE_CONCURRENCY = 1
+export const DEFAULT_REMOVE_CONCURRENCY = 4
+
+export interface BackupCleanOptions {
+  fix?: boolean
+  merge?: boolean
+  remove?: boolean
+  mergeBlockConcurrency?: number
+  mergeConcurrency?: number
+  onProgress?: (progress: { total: number; done: number }) => void
+  logInfo?: (message: any, opts?: object) => void
+  logWarn?: (message: any, opts?: object) => void
+}
+
+export type ResolvedBackupCleanOptions = BackupCleanOptions & {
+  logInfo: (message: any, opts?: object) => void
+  logWarn: (message: any, opts?: object) => void
+}
+
+export interface ArchiveCleanOptions {
+  remove?: boolean
+  merge?: boolean
+  mergedSizes?: Map<string, number>
+}
+
+export interface CheckResult {
+  isValid: boolean
+  // Files referenced by metadata but absent on disk
+  missingDisks?: string[]
+  // Disk files that exist but failed to open (corrupted)
+  brokenDisks?: string[]
+  // Files present on disk but not referenced by metadata
+  orphans?: string[]
+  // Files referenced by at least one valid backup
+  linked?: string[]
+}
+
+export type CleanResult = {
+  removedFiles: string[]
+  merge?: boolean
+  size?: number
+  mergedSizes?: Map<string, number>
+}
+
+export interface VmBackupInterface {
+  handler: RemoteHandlerAbstract
+  // metadataPath: string
+  // metadata: PartialBackupMetadata
+  rootPath: string
+  opts: BackupCleanOptions
+
+  init(): Promise<void>
+  check(): Promise<CheckResult>
+  clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
+  getAssociatedFiles(opts: object): Array<string>
+}

@xen-orchestra/backup-archive/src/VmBackupDirectory.mts

@@ -0,0 +1,289 @@
+import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+import { basename, normalize } from '@xen-orchestra/fs/path'
+import { resolve } from 'node:path'
+import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
+import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
+import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
+import {
+  ArchiveCleanOptions,
+  BackupCleanOptions,
+  CheckResult,
+  CleanResult,
+  VmBackupInterface,
+  PartialBackupMetadata,
+  ResolvedBackupCleanOptions,
+  DEFAULT_MERGE_CONCURRENCY,
+} from './VmBackup.types.mjs'
+import { cleanOrphanDiskDirs } from '@xen-orchestra/backup-archive/disks'
+import { asyncEach } from '@vates/async-each'
+import { createLogger } from '@xen-orchestra/log'
+import { promisify } from 'node:util'
+import zlib from 'node:zlib'
+
+const gzip = promisify(zlib.gzip)
+const gunzip = promisify(zlib.gunzip)
+
+const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
+
+const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
+
+export class VmBackupDirectory implements VmBackupInterface {
+  handler: RemoteHandlerAbstract
+  rootPath: string
+  files: Array<string> = new Array()
+  orphans: Set<string> = new Set()
+  backupArchives: Map<string, VmBackupInterface> = new Map()
+  opts: ResolvedBackupCleanOptions
+
+  // Cached result of the last check() call; invalidated by init()
+  #checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
+  #uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
+
+  // Set by #checkCacheCount(): the on-disk cache did not match the archives found
+  // on disk, so clean() must regenerate it even when nothing was merged/removed.
+  #cacheOutOfSync = false
+
+  constructor(
+    handler: RemoteHandlerAbstract,
+    vmBackupPath: string,
+    opts: BackupCleanOptions = {
+      fix: true,
+      merge: false,
+      remove: false,
+      logInfo,
+      logWarn,
+    }
+  ) {
+    this.handler = handler
+    this.rootPath = vmBackupPath
+    this.opts = {
+      ...opts,
+      fix: opts.fix ?? true,
+      merge: opts.merge ?? false,
+      remove: opts.remove ?? false,
+      logInfo: opts.logInfo ?? logInfo,
+      logWarn: opts.logWarn ?? logWarn,
+    }
+  }
+
+  async #readCache(path: string): Promise<Record<string, unknown> | undefined> {
+    try {
+      return JSON.parse((await gunzip(await this.handler.readFile(path))).toString())
+    } catch (error) {
+      if (error?.code !== 'ENOENT') {
+        logWarn('failed to read cache', { error, path })
+      }
+    }
+  }
+
+  async #writeCache(path: string, data: Record<string, unknown>): Promise<void> {
+    try {
+      await this.handler.writeFile(path, await gzip(JSON.stringify(data)), { flags: 'w' })
+    } catch (error) {
+      logWarn('failed to write cache', { error, path })
+    }
+  }
+
+  async init() {
+    this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
+    this.#checkResult = undefined
+    this.#uniqueLineages = new Map()
+
+    for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
+      let metadata: PartialBackupMetadata | undefined = undefined
+      try {
+        metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
+      } catch (error) {
+        this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
+      }
+      if (metadata !== undefined) {
+        try {
+          const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
+          this.backupArchives.set(fullPath, backupArchive)
+        } catch (error) {
+          if (error?.code === 'NOT_SUPPORTED') throw error
+          this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
+        }
+      }
+    }
+  }
+
+  getAssociatedFiles({ prefix = false }) {
+    const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
+    return prefix ? files : files.map(file => basename(file))
+  }
+
+  async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
+    await this.#checkCacheCount()
+
+    for (const backupArchive of this.backupArchives.values()) {
+      await backupArchive.check()
+    }
+
+    // allUsedFiles is used for root-level orphan detection only.
+    // Active disk paths are accumulated per-lineage by each archive during check() above.
+    const allUsedFiles = new Set<string>([
+      ...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
+      ...this.getAssociatedFiles({ prefix: true }),
+    ])
+
+    for (const lineage of this.#uniqueLineages!.values()) {
+      await lineage.check()
+    }
+
+    const orphans = this.files.filter(file => !allUsedFiles.has(file))
+    const linked = Array.from(allUsedFiles)
+    this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
+    return this.#checkResult
+  }
+
+  async clean({
+    remove = this.opts.remove ?? false,
+    merge = this.opts.merge ?? false,
+  }: ArchiveCleanOptions = {}): Promise<CleanResult> {
+    // Use cached check result if available, otherwise run check now
+    const { orphans } = this.#checkResult ?? (await this.check())
+
+    let cacheNeedsRegen = false
+    let someLineageMergedOrShouldBe = false
+
+    // Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
+    const allMergedSizes = new Map<string, number>()
+    await asyncEach(
+      Array.from(this.#uniqueLineages!.entries()),
+      async ([_vdiDir, lineage]) => {
+        const { mergedSizes, removedFiles, merge: hasPendingMerge } = await lineage.clean({ remove, merge })
+        if (removedFiles.length > 0) {
+          cacheNeedsRegen = true
+        }
+        if (hasPendingMerge) {
+          someLineageMergedOrShouldBe = true
+        }
+        for (const [diskPath, size] of mergedSizes || []) {
+          allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
+        }
+      },
+      { concurrency: DEFAULT_MERGE_CONCURRENCY }
+    )
+
+    // Delete VDI directories not referenced by any archive
+    const coveredDirs = new Set(this.#uniqueLineages!.keys())
+    await cleanOrphanDiskDirs(this.handler, this.rootPath, coveredDirs, {
+      remove,
+      logWarn: this.opts.logWarn,
+      logInfo: this.opts.logInfo,
+    })
+
+    // Let each archive clean its own files (e.g. remove metadata for incomplete backups)
+    // and update metadata with merged sizes if applicable
+    await asyncEach(
+      Array.from(this.backupArchives.values()),
+      async (archive: VmBackupInterface) => {
+        const { removedFiles } = await archive.clean({ remove, mergedSizes: allMergedSizes })
+        if (removedFiles.length > 0) {
+          cacheNeedsRegen = true
+        }
+      },
+      { concurrency: 2 }
+    )
+
+    if (allMergedSizes.size > 0 || cacheNeedsRegen || this.#cacheOutOfSync) {
+      await this.#regenerateCache()
+    }
+
+    // Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
+    if (remove) {
+      await asyncEach(
+        orphans,
+        async (orphan: string) => {
+          try {
+            await this.handler.unlink(orphan)
+          } catch (error) {
+            if (error?.code === 'EISDIR') {
+              this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
+            } else if (error?.code !== 'ENOENT') {
+              throw error
+            }
+          }
+        },
+        { concurrency: 2 }
+      )
+    }
+    const size = [...allMergedSizes.values()].reduce((total, merged) => total + merged, 0)
+    return { removedFiles: orphans, merge: someLineageMergedOrShouldBe, size: size }
+  }
+
+  async #checkCacheCount(): Promise<void> {
+    const cachePath = `${this.rootPath}/cache.json.gz`
+    const existingCache = await this.#readCache(cachePath)
+    const actual = existingCache === undefined ? 0 : Object.keys(existingCache).length
+    const expected = this.backupArchives.size
+    this.#cacheOutOfSync = actual !== expected
+    if (this.#cacheOutOfSync) {
+      this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
+    }
+  }
+
+  async #regenerateCache(): Promise<void> {
+    const cachePath = `${this.rootPath}/cache.json.gz`
+    const cache: Record<string, object> = {}
+    for (const [path, archive] of this.backupArchives.entries()) {
+      const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
+      cache[path] = { _filename: path, id: path, ...a.metadata }
+    }
+    await this.#writeCache(cachePath, cache)
+  }
+
+  async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
+    let backupArchive: VmBackupInterface
+    try {
+      if (metadata.mode === 'full') {
+        backupArchive = new VmFullBackupArchive(
+          this.handler,
+          this.rootPath,
+          metadataPath,
+          metadata,
+          resolve('/', this.rootPath, metadata.xva!),
+          this.opts
+        )
+      } else if (metadata.mode === 'delta') {
+        const rawDiskPaths = metadata.vhds
+        const diskPaths =
+          rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
+        backupArchive = new VmIncrementalBackupArchive(
+          this.handler,
+          this.rootPath,
+          metadataPath,
+          metadata,
+          diskPaths,
+          this.opts,
+          this.#uniqueLineages
+        )
+      } else {
+        throw new Error(`Mode ${metadata.mode} not supported`)
+      }
+    } catch (error) {
+      this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
+      throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
+    }
+    await backupArchive.init()
+    return backupArchive
+  }
+
+  /**
+   * Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
+   * The `lock` option is accepted but ignored (locking is the caller's responsibility).
+   */
+  static async cleanVm(
+    handler: RemoteHandlerAbstract,
+    vmBackupPath: string,
+    opts: BackupCleanOptions & { lock?: boolean } = {}
+  ) {
+    const { lock: _lock, ...cleanOpts } = opts
+    const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
+    await dir.init()
+    await dir.check()
+    const { merge, size } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
+    return { merge, size }
+  }
+}

@xen-orchestra/backup-archive/src/VmFullBackupArchive.integ.mjs

@@ -0,0 +1,157 @@
+import test from 'node:test'
+import { strict as assert } from 'node:assert'
+
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import * as uuid from 'uuid'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { rimraf } from 'rimraf'
+// eslint-disable-next-line n/no-missing-import
+import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
+import tar from 'tar-stream'
+const { beforeEach, afterEach, describe } = test
+
+let tempDir, handler, vmBackupDir
+const vmUuid = 'test-vm-uuid'
+const rootPath = `xo-vm-backups/${vmUuid}`
+
+async function createMinimalXva() {
+  const pack = tar.pack()
+  pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
+  pack.finalize()
+
+  const chunks = []
+  for await (const chunk of pack) {
+    chunks.push(chunk)
+  }
+  return Buffer.concat(chunks)
+}
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  await fs.mkdirp(`${tempDir}/${rootPath}`)
+  vmBackupDir = new VmBackupDirectory(handler, rootPath)
+})
+
+afterEach(async () => {
+  await rimraf(tempDir)
+  await handler.forget()
+})
+
+const uniqueId = () => uuid.v1()
+
+async function createFullBackupMetadata(name, xvaName) {
+  await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
+  const size = await handler.getSize(`${rootPath}/${xvaName}`)
+  const metadata = {
+    mode: 'full',
+    xva: `./${xvaName}`,
+    vm: { uuid: vmUuid },
+    jobId: uniqueId(),
+    timestamp: Date.now(),
+    size,
+  }
+  await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
+  return metadata
+}
+
+describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
+  test('init() loads full backup archives from metadata files', async () => {
+    await createFullBackupMetadata('backup1.json', 'backup1.xva')
+    await createFullBackupMetadata('backup2.json', 'backup2.xva')
+
+    await vmBackupDir.init()
+
+    assert.equal(vmBackupDir.backupArchives.size, 2)
+  })
+
+  test('getAssociatedFiles() returns cache files', async () => {
+    // Create a cache file
+    await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
+    await vmBackupDir.init()
+
+    const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
+    const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
+
+    assert.equal(filesWithPrefix.length, 1)
+    assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
+    assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
+  })
+
+  test('check() identifies orphan files', async () => {
+    // Create a valid backup
+    await createFullBackupMetadata('backup1.json', 'backup1.xva')
+    // Create orphan files (not referenced by any metadata)
+    await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
+    await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
+
+    await vmBackupDir.init()
+    const { orphans, linked } = await vmBackupDir.check()
+
+    assert.ok(orphans.some(f => f.includes('orphan.xva')))
+    assert.ok(orphans.some(f => f.includes('random-file.txt')))
+    assert.ok(linked.some(f => f.includes('backup1.json')))
+    assert.ok(linked.some(f => f.includes('backup1.xva')))
+  })
+
+  test('clean() removes orphan files', async () => {
+    await createFullBackupMetadata('backup1.json', 'backup1.xva')
+    await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
+
+    await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
+
+    const remainingFiles = await handler.list(rootPath)
+    assert.ok(remainingFiles.includes('backup1.json'))
+    assert.ok(remainingFiles.includes('backup1.xva'))
+    assert.ok(!remainingFiles.includes('orphan.xva'))
+  })
+
+  test('Missing XVA cleans the metadata', async () => {
+    await createFullBackupMetadata('metadata.json', 'missing.xva')
+    await handler.unlink(`${rootPath}/missing.xva`)
+    let logged = ''
+    const logInfo = message => {
+      logged += message + '\n'
+    }
+    await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
+    const remainingFiles = await handler.list(rootPath)
+    assert.ok(logged.match(/XVA is missing/g))
+    assert.ok(!remainingFiles.includes('metadata.json'))
+  })
+
+  test('clean() preserves cache.json.gz', async () => {
+    await createFullBackupMetadata('backup1.json', 'backup1.xva')
+    await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
+
+    await VmBackupDirectory.cleanVm(handler, rootPath)
+
+    const remainingFiles = await handler.list(rootPath)
+    assert.equal(remainingFiles.length, 3)
+    assert.ok(remainingFiles.includes('cache.json.gz'))
+  })
+
+  test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
+    await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
+
+    await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true })
+
+    const remainingFiles = await handler.list(rootPath)
+    assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
+  })
+
+  test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
+    await createFullBackupMetadata('backup1.json', 'backup1.xva')
+    await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
+
+    await vmBackupDir.init()
+    await vmBackupDir.clean({ remove: true })
+
+    const remainingFiles = await handler.list(rootPath)
+    assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
+    assert.ok(vmBackupDir.backupArchives.get(`/${rootPath}/backup1.json`).isValid)
+    assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept')
+  })
+})

@xen-orchestra/backup-archive/src/VmFullBackupArchive.mts

@@ -0,0 +1,179 @@
+import { basename, normalize } from '@xen-orchestra/fs/path'
+import assert from 'node:assert'
+import { FileDescriptor } from '@xen-orchestra/fs'
+import {
+  ArchiveCleanOptions,
+  CheckResult,
+  CleanResult,
+  VmBackupInterface,
+  PartialBackupMetadata,
+  ResolvedBackupCleanOptions,
+} from './VmBackup.types.mjs'
+import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+
+const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
+  // https://tools.ietf.org/html/rfc1952.html#page-5
+  Buffer.from('1F8B', 'hex'),
+
+  // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+  Buffer.from('28B52FFD', 'hex'),
+]
+
+const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
+
+const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
+  const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
+  assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
+
+  for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
+    if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
+      return true
+    }
+  }
+  return false
+}
+
+const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
+  if (size <= 1024 || size % 512 !== 0) {
+    return false
+  }
+
+  const buf = Buffer.allocUnsafe(1024)
+  assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
+  return buf.every(byte => byte === 0)
+}
+
+export async function isValidXva(
+  handler: RemoteHandlerAbstract,
+  filePath: string,
+  metadataSize?: number
+): Promise<boolean> {
+  // size is longer when encrypted + reading part of an encrypted file is not implemented
+  if (handler.isEncrypted) {
+    const size = await handler.getSizeOnDisk(filePath)
+    return size > 20
+  }
+
+  let isValid = true
+
+  const fd = await handler.openFile(filePath, 'r')
+  try {
+    const size = await handler.getSize(filePath)
+    if (size < 20) {
+      // neither a valid gzip nor tar
+      return false
+    }
+    let validSize = true
+    if (metadataSize !== undefined) {
+      validSize = metadataSize === size
+    }
+
+    const validDisk = (await isCompressedFile(handler, fd))
+      ? true // compressed files cannot be validated at this time
+      : await isValidTar(handler, size, fd)
+    isValid = validSize && validDisk
+  } finally {
+    handler.closeFile(fd).catch(noop)
+  }
+  return isValid
+}
+
+const noop = (): void => {}
+
+export class VmFullBackupArchive implements VmBackupInterface {
+  handler: RemoteHandlerAbstract
+  rootPath: string
+  xvaPath: string
+  isValid?: boolean
+  missingDisk = false
+  metadataPath: string
+  metadata: PartialBackupMetadata
+  opts: ResolvedBackupCleanOptions
+
+  constructor(
+    handler: RemoteHandlerAbstract,
+    rootPath: string, // xo-vm-backups/<vmUuid>/
+    metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
+    metadata: PartialBackupMetadata,
+    xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
+    opts: ResolvedBackupCleanOptions
+  ) {
+    this.handler = handler
+    this.rootPath = normalize(rootPath)
+    this.metadataPath = normalize(metadataPath)
+    this.metadata = metadata
+    this.xvaPath = normalize(xvaPath)
+    this.opts = opts
+  }
+
+  async init() {}
+
+  /**
+   * As we are not 100% sure if XVA is valid,
+   * we prefer to warn if it is the case but not remove it.
+   */
+  async check(): Promise<CheckResult> {
+    if (this.isValid === undefined) {
+      try {
+        this.isValid = await isValidXva(this.handler, this.xvaPath, this.metadata.size)
+      } catch (error) {
+        this.isValid = false
+        if (error?.code === 'ENOENT') {
+          this.missingDisk = true
+        }
+        this.opts.logWarn('Issue while checking XVA', { error })
+      }
+      if (!this.handler.isEncrypted) {
+        try {
+          await this.handler.getSize(`${this.xvaPath}.checksum`)
+        } catch (error) {
+          this.opts.logWarn('Checksum file not valid, not blocking', { error })
+        }
+      }
+    }
+    if (!this.isValid) {
+      if (this.missingDisk) {
+        this.opts.logWarn('XVA is missing', { path: this.xvaPath })
+      } else {
+        this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
+      }
+    }
+    return { isValid: this.isValid }
+  }
+
+  async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
+    if (this.isValid === undefined) {
+      await this.check()
+    }
+    const removedFiles: string[] = []
+
+    if (!this.isValid && this.missingDisk) {
+      removedFiles.push(this.metadataPath, `${this.xvaPath}.checksum`)
+    }
+
+    if (remove) {
+      if (!this.isValid) {
+        if (!this.missingDisk) {
+          this.opts.logWarn(`This files may be corrupted but not yet to be removed`, {
+            files: this.getAssociatedFiles({ prefix: false }),
+          })
+        } else {
+          for (const file of removedFiles) {
+            try {
+              await this.handler.unlink(file)
+            } catch (error) {
+              this.opts.logWarn(`Issue removing ${file}`, { error })
+            }
+          }
+        }
+      }
+    }
+
+    return { removedFiles, merge: false }
+  }
+
+  getAssociatedFiles({ prefix = false }): Array<string> {
+    let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
+    return prefix ? validFiles : validFiles.map(file => basename(file))
+  }
+}

@xen-orchestra/backup-archive/src/VmIncrementalBackupArchive.integ.mjs

@@ -0,0 +1,859 @@
+import test from 'node:test'
+import { strict as assert } from 'node:assert'
+
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import * as uuid from 'uuid'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+// eslint-disable-next-line n/no-missing-import
+import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
+import { RemoteVhdDisk } from '../dist/disks/RemoteVhdDisk.mjs'
+import { MergeRemoteDisk } from '../dist/disks/MergeRemoteDisk.mjs'
+import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
+import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
+import { dirname, basename } from 'node:path'
+import { promisify } from 'node:util'
+import zlib from 'node:zlib'
+import { rimraf } from 'rimraf'
+
+const { beforeEach, afterEach, describe } = test
+
+const gunzip = promisify(zlib.gunzip)
+
+let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
+const rootPath = 'xo-vm-backups/VMUUID/'
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  jobId = uniqueId()
+  vdiId = uniqueId()
+  relativePath = `vdis/${jobId}/${vdiId}`
+  basePath = `${rootPath}/${relativePath}`
+  vdiId2 = uniqueId()
+  relativePath2 = `vdis/${jobId}/${vdiId2}`
+  basePath2 = `${rootPath}/${relativePath2}`
+  await fs.mkdirp(`${tempDir}/${basePath}`)
+  await fs.mkdirp(`${tempDir}/${basePath2}`)
+})
+
+afterEach(async () => {
+  await rimraf(tempDir)
+  await handler.forget()
+})
+
+const uniqueId = () => uuid.v1()
+const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
+
+async function generateVhd(path, opts = {}) {
+  let vhd
+
+  let dataPath = path
+  if (opts.useAlias) {
+    await handler.mkdir(dirname(path) + '/data/')
+    dataPath = dirname(path) + '/data/' + basename(path)
+  }
+  if (opts.mode === 'directory') {
+    await handler.mkdir(dataPath)
+    vhd = new VhdDirectory(handler, dataPath)
+  } else {
+    const fd = await handler.openFile(dataPath, 'wx')
+    vhd = new VhdFile(handler, fd)
+  }
+
+  vhd.header = { ...VHDHEADER, ...opts.header }
+  vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
+
+  if (vhd.header.parentUuid) {
+    vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
+  } else {
+    vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
+  }
+
+  if (opts.useAlias === true) {
+    await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
+  }
+
+  if (opts.blocks) {
+    for (const blockId of opts.blocks) {
+      await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
+    }
+  }
+  await vhd.writeBlockAllocationTable()
+  await vhd.writeHeader()
+  await vhd.writeFooter()
+  return vhd
+}
+
+test('It remove broken vhd', async () => {
+  // todo also tests a directory and an alias
+
+  await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
+  // metadata references a non-existent disk so the lineage is created for this vdiDir
+  // and notReallyAVhd.vhd is treated as an unreferenced orphan
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
+  )
+  assert.equal((await handler.list(basePath)).length, 1)
+  let logged = ''
+  const logInfo = message => {
+    logged += message
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
+  assert.ok(logged.includes('failed to open disk'))
+  // not removed
+  assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
+  // really remove it
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
+  assert.deepEqual(await handler.list(basePath), [])
+})
+
+test('it remove vhd with missing or multiple ancestors', async () => {
+  // one with a broken parent, should be deleted
+  await generateVhd(`${basePath}/abandoned.vhd`, {
+    header: {
+      parentUnicodeName: 'gone.vhd',
+      parentUuid: uniqueIdBuffer(),
+    },
+  })
+
+  // one orphan, which is a full vhd, no parent : should stay
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan in the metadata : should stay
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUuid: orphan.footer.uuid,
+    },
+  })
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
+    })
+  )
+  // clean
+  let logged = ''
+  const logInfo = message => {
+    logged += message + '\n'
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
+
+  // abandoned has a missing parent, orphan and child have no active backup reference
+  const detectedMissingParent = logged.match(/parent disk is missing/g) || []
+  assert.equal(detectedMissingParent.length, 1)
+
+  // we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
+})
+
+test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
+  // create a metadata file marking child and orphan as ok
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${relativePath}/orphan.vhd`,
+        `${relativePath}/child.vhd`,
+        // abandoned.json is not here
+      ],
+    })
+  )
+
+  await generateVhd(`${basePath}/abandoned.vhd`)
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUuid: orphan.footer.uuid,
+    },
+  })
+
+  let logged = ''
+  const logInfo = message => {
+    logged += message + '\n'
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
+  let matched = logged.match(/deleting unused disk/g) || []
+  assert.equal(matched.length, 1) // only one vhd should have been deleted
+
+  // a missing vhd cause clean to remove all vhds
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${relativePath}/deleted.vhd`, // in metadata but missing from disk
+        `${relativePath}/orphan.vhd`,
+        `${relativePath}/child.vhd`,
+        // abandoned.vhd is not here anymore
+      ],
+    }),
+    { flags: 'w' }
+  )
+  logged = ''
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
+  matched = logged.match(/deleting unused disk/g) || []
+  assert.equal(matched.length, 2) // all vhds (orphan and  child  ) should have been deleted
+
+  assert.ok(
+    (await handler.list(rootPath)).includes('cache.json.gz'),
+    'cache.json.gz should be regenerated after metadata deletion'
+  )
+})
+
+test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
+  // A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
+  await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
+
+  // Metadata references the corrupt VHD as if it were a valid active disk
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [`${relativePath}/broken.vhd`],
+    })
+  )
+
+  let logged = ''
+  const logInfo = message => {
+    logged += message
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, {
+    remove: true,
+    logInfo: () => {},
+    logWarn: logInfo,
+  })
+
+  assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
+  assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
+
+  // metadata removed because backup is incomplete
+  assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
+
+  // broken VHD removed via the orphan path in lineage.clean()
+  assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
+})
+
+test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
+  // Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
+  // Each snapshot is the active disk for exactly one backup metadata file.
+  //
+  // Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
+  // own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
+  // RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
+  // their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
+  // deleted — destroying two live backups.
+
+  const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
+  const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
+    header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
+  })
+  await generateVhd(`${basePath}/snapshot3.vhd`, {
+    header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
+  })
+
+  await handler.writeFile(
+    `${rootPath}/metadata1.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
+  )
+  await handler.writeFile(
+    `${rootPath}/metadata2.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
+  )
+  await handler.writeFile(
+    `${rootPath}/metadata3.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
+
+  const remaining = await handler.list(basePath)
+  assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
+  assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
+  assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
+  assert.equal(remaining.length, 3, 'no extra files should remain')
+
+  // All three metadata files must also survive (all backups are complete)
+  const rootFiles = await handler.list(rootPath)
+  assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
+  assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
+  assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
+})
+
+test('it handles multiple VDI directories independently', async () => {
+  // vdiDir1: diskA1 (base) <- diskA2, plus an orphan
+  const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
+  await generateVhd(`${basePath}/diskA2.vhd`, {
+    header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
+  })
+  await generateVhd(`${basePath}/orphanA.vhd`)
+
+  // vdiDir2: diskB1 (base) <- diskB2, plus an orphan
+  const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
+  await generateVhd(`${basePath2}/diskB2.vhd`, {
+    header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
+  })
+  await generateVhd(`${basePath2}/orphanB.vhd`)
+
+  // archive1 covers both base disks, archive2 covers both child disks
+  await handler.writeFile(
+    `${rootPath}/metadata1.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
+  )
+  await handler.writeFile(
+    `${rootPath}/metadata2.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
+
+  const remaining1 = await handler.list(basePath)
+  assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
+  assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
+  assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
+
+  const remaining2 = await handler.list(basePath2)
+  assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
+  assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
+  assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
+})
+
+test('it merges delta of non destroyed chain', async () => {
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 12000, // a size too small
+      vhds: [
+        `${relativePath}/grandchild.vhd`, // grand child should not be merged
+        `${relativePath}/child.vhd`,
+        // orphan is not here, he should be merged in child
+      ],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUuid: orphan.footer.uuid,
+    },
+    blocks: [0, 1],
+  })
+  // a grand child
+  await generateVhd(`${basePath}/grandchild.vhd`, {
+    header: {
+      parentUnicodeName: 'child.vhd',
+      parentUuid: child.footer.uuid,
+    },
+  })
+
+  let logged = []
+  const logInfo = message => {
+    logged.push(message)
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
+  assert.equal(logged[0], `unexpected number of entries in backup cache`)
+
+  logged = []
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
+  assert.equal(logged.includes(`merging disk chain`), true)
+
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  assert.equal(remainingVhds.length, 2)
+  assert.equal(remainingVhds.includes('child.vhd'), true)
+  assert.equal(remainingVhds.includes('grandchild.vhd'), true)
+
+  const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
+  assert.equal(metadata.size, 4299776, 'metadata.size should be updated after merge')
+})
+
+test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
+  // ancestor -> child -> grandchild
+  // only grandchild is referenced in metadata: both ancestor and child are orphans
+  // they must be merged as a single 3-disk chain, not processed separately
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 12000,
+      vhds: [`${relativePath}/grandchild.vhd`],
+    })
+  )
+
+  const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
+  })
+  await generateVhd(`${basePath}/grandchild.vhd`, {
+    header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
+  })
+
+  const mergeChains = []
+  const logInfo = (message, opts) => {
+    if (message === 'merging disk chain') mergeChains.push(opts.chain)
+  }
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
+
+  // all three disks must be handled in exactly one merge, not two separate operations
+  assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
+  assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
+
+  const remainingVhds = await handler.list(basePath)
+  assert.equal(remainingVhds.includes('grandchild.vhd'), true)
+  assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
+  assert.equal(remainingVhds.includes('child.vhd'), false)
+})
+
+test('it resumes, through cleanVm, an interrupted merge whose ancestor lost its footer', async () => {
+  // Reproduces the production "footer1 !== footer2" failure end-to-end via the full cleanVm
+  // pipeline. ancestor (full) <- child (differencing, the active backup).
+  const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
+    blocks: [2, 3, 4, 5],
+  })
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', size: 12000, vhds: [`${relativePath}/child.vhd`] })
+  )
+
+  // Simulate a merge killed mid-transfer: write 2 of the 4 child blocks into the ancestor
+  // (which overwrites its end footer) then throw before mergeMetadata() rewrites the footer.
+  // This leaves the ancestor footerless AND a `.merge.json` state file — the exact post-crash
+  // state that makes cleanVm take the resume path.
+  const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
+  const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
+  await parent.init({ force: false })
+  await child.init({ force: false })
+  const merger = new MergeRemoteDisk(handler, { removeUnused: true, writeStateDelay: 0 })
+  let written = 0
+  const realWriteBlock = parent.writeBlock.bind(parent)
+  parent.writeBlock = async diskBlock => {
+    if (written === 2) {
+      throw new Error('simulated interruption')
+    }
+    written++
+    return realWriteBlock(diskBlock)
+  }
+  await assert.rejects(merger.merge(parent, child), /simulated interruption/)
+
+  assert.ok(
+    (await handler.list(basePath)).includes('.ancestor.vhd.merge.json'),
+    'an interrupted-merge state file should remain'
+  )
+
+  // Full pipeline: cleanVm must recognize the interrupted merge and resume it — opening the
+  // footerless ancestor through the chain without throwing "footer1 !== footer2".
+  await VmBackupDirectory.cleanVm(handler, rootPath, {
+    remove: true,
+    merge: true,
+    logInfo: () => {},
+    logWarn: () => {},
+  })
+
+  // The merge completed: the ancestor was consolidated into child.vhd and the state file is gone.
+  const remaining = await handler.list(basePath)
+  assert.equal(remaining.includes('.ancestor.vhd.merge.json'), false, 'state file removed after successful resume')
+  assert.equal(remaining.includes('ancestor.vhd'), false, 'ancestor consolidated into the merge target')
+  assert.equal(remaining.includes('child.vhd'), true, 'merge target remains')
+
+  // No blocks lost: the resumed merge wrote the blocks (4, 5) that were never written before
+  // the crash, so the consolidated disk holds the full set.
+  const merged = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
+  await merged.init({ force: false })
+  try {
+    assert.deepEqual(merged.getBlockIndexes(), [0, 1, 2, 3, 4, 5], 'all blocks present after resumed merge')
+  } finally {
+    await merged.close()
+  }
+
+  // The backup is consistent, so its metadata is preserved.
+  assert.ok((await handler.list(rootPath)).includes('metadata.json'), 'metadata should be preserved')
+})
+
+test('it does not warn "missing target of alias" for aliases deleted by merge', async () => {
+  // Regression: RemoteDiskLineage.#cleanOrphanDataFiles iterated #diskPaths entries just
+  // deleted during the same clean() call's merge phase, producing false "missing target of
+  // alias" warnings
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`, {
+    useAlias: true,
+    blocks: [0],
+  })
+  await generateVhd(`${basePath}/child.vhd`, {
+    useAlias: true,
+    header: {
+      parentUnicodeName: 'orphan.vhd.alias.vhd',
+      parentUuid: orphan.footer.uuid,
+    },
+    blocks: [1],
+  })
+
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [`${relativePath}/child.vhd.alias.vhd`],
+    })
+  )
+
+  const warnings = []
+  await VmBackupDirectory.cleanVm(handler, rootPath, {
+    remove: true,
+    merge: true,
+    logWarn: (message, data) => warnings.push({ message, data }),
+    logInfo: () => {},
+  })
+
+  const missingTargetWarnings = warnings.filter(w => w.message === 'missing target of alias')
+  assert.equal(
+    missingTargetWarnings.length,
+    0,
+    `"missing target of alias" must not fire after merge; got: ${JSON.stringify(missingTargetWarnings)}`
+  )
+
+  const remaining = await handler.list(basePath)
+  assert.ok(!remaining.includes('orphan.vhd.alias.vhd'), 'orphan alias must be deleted after merge')
+  assert.ok(remaining.includes('child.vhd.alias.vhd'), 'child alias must survive as merge target')
+})
+
+test('it resumes an interrupted merge with chain field', async () => {
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
+  )
+
+  const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
+  const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
+    header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
+    blocks: [5, 6],
+  })
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
+    blocks: [7, 8],
+  })
+
+  await handler.writeFile(
+    `${basePath}/.ancestor.vhd.merge.json`,
+    JSON.stringify({
+      parent: { uuid: '0' },
+      child: { uuid: '0' },
+      chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
+      currentBlock: 3,
+      mergedDataSize: 3 * 2 * 1024 * 1024,
+      step: 'mergeBlocks',
+      diskSize: 0,
+    })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
+
+  const files = await handler.list(basePath)
+  assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
+  assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
+  assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
+  assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
+})
+
+test('it resumes an interrupted merge without chain field', async () => {
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 209920,
+      vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUuid: orphan.footer.uuid,
+    },
+  })
+  // a merge in progress file
+  await handler.writeFile(
+    `${basePath}/.orphan.vhd.merge.json`,
+    JSON.stringify({
+      parent: {
+        header: orphan.header.checksum,
+      },
+      child: {
+        header: child.header.checksum,
+      },
+    })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  assert.equal(remainingVhds.length, 1)
+  assert.equal(remainingVhds.includes('child.vhd'), true)
+})
+
+// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
+// the message and resulting files should be identical to the output with vhd files which is tested independently
+describe('tests multiple combination ', { concurrency: 1 }, () => {
+  for (const useAlias of [true, false]) {
+    for (const vhdMode of ['file', 'directory']) {
+      test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
+        // a broken VHD
+        if (useAlias) {
+          await handler.mkdir(basePath + '/data')
+        }
+
+        const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
+
+        if (vhdMode === 'directory') {
+          await handler.mkdir(brokenVhdDataPath)
+        } else {
+          await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
+        }
+        if (useAlias) {
+          await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
+        }
+
+        // a vhd non referenced in metadata
+        await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
+        // an abandoned delta vhd without its parent
+        await generateVhd(`${basePath}/abandoned.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'gone.vhd',
+            parentUuid: uniqueIdBuffer(),
+          },
+        })
+
+        // an ancestor of a vhd present in metadata
+        const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          blocks: [1, 3],
+        })
+        const child = await generateVhd(`${basePath}/child.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUuid: ancestor.footer.uuid,
+          },
+          blocks: [1, 2],
+        })
+        // a grand child  vhd in metadata
+        await generateVhd(`${basePath}/grandchild.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUuid: child.footer.uuid,
+          },
+          blocks: [2, 3],
+        })
+
+        // an older parent that was merging in clean
+        const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        // a clean  vhd in metadata
+        const clean = await generateVhd(`${basePath}/clean.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUuid: cleanAncestor.footer.uuid,
+          },
+        })
+
+        await handler.writeFile(
+          `${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
+          JSON.stringify({
+            parent: {
+              header: cleanAncestor.header.checksum,
+            },
+            child: {
+              header: clean.header.checksum,
+            },
+          })
+        )
+
+        // the metadata file
+        await handler.writeFile(
+          `${rootPath}/metadata.json`,
+          JSON.stringify({
+            mode: 'delta',
+            vhds: [
+              `${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
+              `${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
+              `${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
+            ],
+          })
+        )
+        if (!useAlias && vhdMode === 'directory') {
+          try {
+            await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
+          } catch (err) {
+            assert.strictEqual(
+              err.code,
+              'NOT_SUPPORTED',
+              'Merging directory without alias should raise a not supported error'
+            )
+            return
+          }
+          assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
+        }
+        await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
+
+        const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
+        // size should be the size of children + grand children + clean after the merge
+        assert.deepEqual(metadata.size, 6501888)
+        // cache.json.gz must be regenerated when metadata changes after a merge
+        assert.ok(
+          (await handler.list(rootPath)).includes('cache.json.gz'),
+          'cache.json.gz should be regenerated after a merge'
+        )
+
+        // broken vhd, non referenced, abandoned should be deleted ( alias and data)
+        // ancestor and child should be merged
+        // grand child and clean vhd should not have changed
+        const survivors = await handler.list(basePath)
+        if (useAlias) {
+          const dataSurvivors = await handler.list(basePath + '/data')
+          // the goal of the alias : do not move a full folder
+          assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
+          assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
+          assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
+          assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
+          assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
+          assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
+          assert.equal(survivors.length, 4) // the 3 ok + data
+          assert.equal(dataSurvivors.length, 3)
+        } else {
+          assert.equal(survivors.includes('clean.vhd'), true)
+          assert.equal(survivors.includes('child.vhd'), true)
+          assert.equal(survivors.includes('grandchild.vhd'), true)
+          assert.equal(survivors.length, 3)
+        }
+      })
+    }
+  }
+})
+test('it cleans orphan merge states ', async () => {
+  await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
+  // metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
+
+  assert.deepEqual(await handler.list(basePath), [])
+})
+
+test('check all types of aliases, corrupted, missing or not', async () => {
+  // valid alias pointing to an existing data file
+  await handler.mkdir(`${basePath}/data`)
+  await generateVhd(`${basePath}/data/ok.vhd`)
+  await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
+
+  // broken alias pointing to a missing data file
+  await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
+
+  // alias pointing to an existing but corrupt data file
+  await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
+  await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
+
+  // data file without alias
+  await generateVhd(`${basePath}/data/missingalias.vhd`)
+
+  // only the valid alias is referenced by a complete backup
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
+  )
+
+  await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
+
+  // only ok.alias.vhd has survived
+  const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
+  assert.equal(aliases.length, 1)
+  assert.equal(aliases[0], 'ok.alias.vhd')
+
+  // corrupt.vhd and missingalias.vhd must be gone
+  const data = await handler.list(`${basePath}/data`)
+  assert.equal(data.length, 1)
+  assert.equal(data[0], 'ok.vhd')
+})
+
+test('it preserves VDI directory when handler.list() throws during lineage init', async () => {
+  // Regression: when handler.list(vdiDir) threw (e.g. S3 404), the lineage
+  // was never added to #uniqueLineages, so cleanOrphanDiskDirs treated the directory
+  // as an orphan and deleted it with rmtree()
+  await generateVhd(`${basePath}/snapshot.vhd`)
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot.vhd`] })
+  )
+
+  const originalList = handler.list.bind(handler)
+  handler.list = async (path, opts) => {
+    if (typeof path === 'string' && path.includes(vdiId)) {
+      throw Object.assign(new Error('Simulated transient storage error'), { code: 'ENOENT' })
+    }
+    return originalList(path, opts)
+  }
+
+  try {
+    await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
+  } finally {
+    handler.list = originalList
+  }
+
+  const remaining = await handler.list(basePath)
+  assert.ok(remaining.includes('snapshot.vhd'), 'VHD must survive when handler.list() throws during lineage init')
+})
+
+test('it regenerates the cache when its entry count is out of sync, even with nothing to merge/remove', async () => {
+  // a referenced disk + its metadata, but no cache.json.gz on disk
+  await generateVhd(`${basePath}/diskA1.vhd`)
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`] })
+  )
+  await assert.rejects(handler.readFile(`${rootPath}/cache.json.gz`), { code: 'ENOENT' })
+
+  const logged = []
+  // remove/merge both false: cacheNeedsRegen stays false and no merge happens,
+  // so only the cache-count mismatch can trigger the regeneration
+  await VmBackupDirectory.cleanVm(handler, rootPath, {
+    remove: false,
+    merge: false,
+    logInfo: () => {},
+    logWarn: message => logged.push(message),
+  })
+
+  assert.ok(logged.includes('unexpected number of entries in backup cache'), 'mismatch should be reported')
+
+  const cache = JSON.parse((await gunzip(await handler.readFile(`${rootPath}/cache.json.gz`))).toString())
+  assert.equal(Object.keys(cache).length, 1, 'cache should be regenerated to match the archives on disk')
+})

@xen-orchestra/backup-archive/src/VmIncrementalBackupArchive.mts

@@ -0,0 +1,174 @@
+import {
+  ArchiveCleanOptions,
+  CheckResult,
+  CleanResult,
+  ResolvedBackupCleanOptions,
+  VmBackupInterface,
+  PartialBackupMetadata,
+} from './VmBackup.types.mjs'
+import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
+import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
+
+export class VmIncrementalBackupArchive implements VmBackupInterface {
+  handler: RemoteHandlerAbstract
+  metadataPath: string
+  metadata: PartialBackupMetadata
+  readonly diskPaths: Array<string>
+  rootPath: string
+  opts: ResolvedBackupCleanOptions
+  diskLineages: Map<string, RemoteDiskLineage> = new Map()
+
+  #isChecked = false
+  #isComplete = false
+
+  get isChecked(): boolean {
+    return this.#isChecked
+  }
+
+  get isComplete(): boolean {
+    return this.#isComplete
+  }
+
+  constructor(
+    handler: RemoteHandlerAbstract,
+    rootPath: string, // xo-vm-backups/<vmUuid>/
+    metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
+    metadata: PartialBackupMetadata,
+    diskPaths: Array<string>, // xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/<snapshotUuid>.alias.vhd (one per VDI)
+    opts: ResolvedBackupCleanOptions,
+    lineageRegistry?: Map<string, RemoteDiskLineage>
+  ) {
+    this.handler = handler
+    this.rootPath = normalize(rootPath)
+    this.metadataPath = normalize(metadataPath)
+    this.metadata = metadata
+    this.diskPaths = diskPaths.map(path => normalize(path))
+    this.opts = opts
+    if (lineageRegistry !== undefined) {
+      this.diskLineages = lineageRegistry
+    }
+  }
+
+  async init(): Promise<void> {
+    // Build one RemoteDiskLineage per VDI directory derived from diskPaths
+    for (const diskPath of this.diskPaths) {
+      const vdiDir = dirname(diskPath)
+      if (!this.diskLineages.has(vdiDir)) {
+        const lineage = new RemoteDiskLineage(this.handler, vdiDir, this.opts)
+        this.diskLineages.set(vdiDir, lineage)
+        try {
+          await lineage.init()
+        } catch (error) {
+          if (error?.code === 'NOT_SUPPORTED') throw error
+          this.opts.logWarn('failed to scan VDI directory', { vdiDir, error })
+        }
+      }
+    }
+  }
+
+  async check(): Promise<CheckResult> {
+    const missingDisks: string[] = []
+    const brokenDisks: string[] = []
+
+    if (this.diskPaths.length === 0) {
+      this.opts.logWarn('incremental backup has no disk paths', { metadataPath: this.metadataPath })
+      this.#isComplete = false
+    } else {
+      for (const diskPath of this.diskPaths) {
+        const lineage = this.diskLineages.get(dirname(diskPath))
+        if (lineage === undefined || !lineage.isPathAccessible(diskPath)) {
+          if (lineage?.isBroken(diskPath)) {
+            this.opts.logWarn('disk check error', { path: diskPath })
+            brokenDisks.push(diskPath)
+          } else {
+            this.opts.logWarn('disk file missing', { path: diskPath })
+            missingDisks.push(diskPath)
+          }
+        }
+      }
+
+      this.#isComplete = missingDisks.length === 0 && brokenDisks.length === 0
+      if (!this.#isComplete) {
+        this.opts.logWarn('incremental backup is incomplete', {
+          metadataPath: this.metadataPath,
+          missingDisks,
+          brokenDisks,
+        })
+      }
+    }
+    if (this.#isComplete) {
+      for (const diskPath of this.diskPaths) {
+        this.diskLineages.get(dirname(diskPath))?.addActiveDiskPath(diskPath)
+      }
+    }
+
+    this.#isChecked = true
+    return { isValid: this.#isComplete, missingDisks, brokenDisks }
+  }
+
+  /**
+   * Removes the metadata file if the backup is incomplete (missing disks).
+   * If mergedSizes is provided, updates metadata with the total merged size for this archive's disks.
+   * Actual disk merge/deletion is handled by RemoteDiskLineage.
+   */
+  async clean({ remove = this.opts.remove ?? false, mergedSizes }: ArchiveCleanOptions = {}): Promise<CleanResult> {
+    if (!this.#isChecked) {
+      await this.check()
+    }
+
+    const removedFiles: string[] = []
+    if (!this.#isComplete) {
+      removedFiles.push(this.metadataPath)
+    }
+
+    if (remove) {
+      for (const file of removedFiles) {
+        try {
+          await this.handler.unlink(file)
+        } catch (error) {
+          this.opts.logWarn(`Issue removing ${file}`, { error })
+        }
+      }
+    }
+
+    let mergedSize = 0
+    if (mergedSizes !== undefined) {
+      for (const diskPath of this.diskPaths) {
+        mergedSize += mergedSizes.get(diskPath) ?? 0
+      }
+      if (mergedSize > 0) {
+        await this.updateMetadata(mergedSize)
+      }
+    }
+
+    return { removedFiles }
+  }
+
+  /**
+   * Updates metadata.size after a merge and writes the updated metadata file to disk.
+   * Also resets isVhdDifferencing so all VDIs are marked as full (non-differencing).
+   */
+  async updateMetadata(mergedSize: number): Promise<void> {
+    this.metadata.size = mergedSize
+    if (this.metadata.vdis !== undefined) {
+      this.metadata.isVhdDifferencing = {}
+      for (const id of Object.keys(this.metadata.vdis)) {
+        this.metadata.isVhdDifferencing[id] = false
+      }
+    }
+    await this.handler.writeFile(this.metadataPath, JSON.stringify(this.metadata), { flags: 'w' })
+  }
+
+  /**
+   * Returns the metadata file path and active disk paths if the backup is complete, empty otherwise.
+   * Must be called after check().
+   */
+  getAssociatedFiles({ prefix = false }): Array<string> {
+    if (!this.#isComplete) {
+      return []
+    }
+    const files = [this.metadataPath, ...this.diskPaths]
+    return prefix ? files : files.map(file => basename(file))
+  }
+}

@xen-orchestra/backup-archive/src/disks/MergeRemoteDisk.mjs

@@ -2,7 +2,7 @@
 
 /**
  * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
- * @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
+ * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
  */
 
 import assert from 'assert'
@@ -10,15 +10,15 @@ import { createLogger } from '@xen-orchestra/log'
 
 import { basename, dirname } from 'path'
 import { asyncEach } from '@vates/async-each'
-import { relativeFromFile } from '@xen-orchestra/fs/path'
+import { relativeFromFile, normalize, resolveFromFile } from '@xen-orchestra/fs/path'
 
 // @ts-ignore
 const { warn } = createLogger('remote-disk:merge')
 
 /**
  * @typedef {Object} MergeState
- * @property {{ uuid: string }} child
- * @property {{ uuid: string }} parent
+ * @property {{ uuid?: string }} child
+ * @property {{ uuid?: string }} parent
  * @property { string[]  | undefined} chain
  * @property {number} currentBlock
  * @property {number} mergedDataSize
@@ -82,12 +82,12 @@ export class MergeRemoteDisk {
   #lastStateWrittenAt
 
   /**
-   * @type {FileAccessor}
+   * @type {RemoteHandlerAbstract}
    */
   #handler
 
   /**
-   * @param {FileAccessor} handler
+   * @param {RemoteHandlerAbstract} handler
    * @param {Object} params
    * @param {Function} [params.onProgress]
    * @param {Logger | Function} [params.logInfo]
@@ -142,10 +142,11 @@ export class MergeRemoteDisk {
       const mergeStateContent = await this.#handler.readFile(this.#statePath)
       this.#state = JSON.parse(mergeStateContent)
 
-      // work-around a bug introduce in 97d94b795
+      // work-around to bugs introduced in 97d94b795 and 4e50858
       //
       // currentBlock could be `null` due to the JSON.stringify of a `NaN` value
-      if (this.#state?.currentBlock === null) this.#state.currentBlock = 0
+      // could also be -1 due to being set as Math.min(...merging) - 1 if merging contains 0
+      if (this.#state?.currentBlock === null || this.#state?.currentBlock < 0) this.#state.currentBlock = 0
       this.#isResuming = true
     } catch (error) {
       // @ts-ignore
@@ -222,6 +223,7 @@ export class MergeRemoteDisk {
     await this.#mergeBlocks(parentDisk, childDisk)
     await parentDisk.flushMetadata(childDisk)
     await parentDisk.mergeMetadata(childDisk)
+    this.#state.diskSize = parentDisk.getSizeOnDisk()
   }
 
   /**
@@ -254,7 +256,7 @@ export class MergeRemoteDisk {
         const blockSize = await parentDisk.mergeBlock(childDisk, blockId, this.#isResuming)
         this.#state.mergedDataSize += blockSize
 
-        this.#state.currentBlock = Math.min(...merging) - 1
+        this.#state.currentBlock = Math.max(0, Math.min(...merging) - 1)
         merging.delete(blockId)
 
         this.#onProgress({ total: nBlocks, done: counter + 1 })
@@ -266,8 +268,6 @@ export class MergeRemoteDisk {
 
     await this.#writeState()
 
-    this.#state.diskSize = childDisk.getSizeOnDisk()
-
     this.#onProgress({ total: nBlocks, done: nBlocks })
   }
 
@@ -322,4 +322,44 @@ export class MergeRemoteDisk {
 
     return { mergedDataSize, finalDiskSize }
   }
+
+  /**
+   * Scans vdiDir files for interrupted merge state files and resolves their chains
+   *
+   * @param {RemoteHandlerAbstract} handler
+   * @param {string} vdiDir
+   * @param {string[]} files - all file paths in vdiDir (prependDir: true)
+   * @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
+   * @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
+   * @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
+   */
+  static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
+    const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
+    const result = new Map()
+
+    for (const filePath of files) {
+      const match = STATE_FILE_RE.exec(basename(filePath))
+      if (match === null) continue
+
+      const parentPath = normalize(vdiDir + '/' + match[1])
+      let chain
+      try {
+        const state = JSON.parse(await handler.readFile(filePath))
+        if (Array.isArray(state?.chain)) {
+          chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
+        } else {
+          // old state file without chain: fall back to UUID lookup then header-based map
+          const childUuid = state?.child?.uuid
+          const childPath = childUuid !== undefined ? uuidToPath.get(childUuid) : childOf.get(parentPath)
+          if (childPath !== undefined) chain = [parentPath, childPath]
+        }
+      } catch (error) {
+        warn("Merge state unreadable, can't restart merging.", error)
+      }
+
+      result.set(parentPath, { stateFilePath: filePath, chain })
+    }
+
+    return result
+  }
 }

@xen-orchestra/backup-archive/src/disks/RemoteDisk.mjs

@@ -73,6 +73,30 @@ export class RemoteDisk extends RandomAccessDisk {
     throw new Error(`getUuid must be implemented`)
   }
 
+  /**
+   * Abstract
+   * @returns {string}
+   */
+  getParentUuid() {
+    throw new Error(`getParentUuid must be implemented`)
+  }
+
+  /**
+   * Abstract
+   * @returns {string}
+   */
+  getParentPath() {
+    throw new Error(`getParentPath must be implemented`)
+  }
+
+  /**
+   * Abstract
+   * @returns {object}
+   */
+  getMetadata() {
+    throw new Error(`getMetadata must be implemented`)
+  }
+
   /**
    * Abstract
    * @returns {Promise<boolean>} canMergeConcurently
@@ -88,6 +112,18 @@ export class RemoteDisk extends RandomAccessDisk {
     return Math.ceil(this.getVirtualSize() / this.getBlockSize())
   }
 
+  /**
+   * Returns true if every block present in `other` is also present in this disk.
+   * @param {RemoteDisk} other
+   * @returns {boolean}
+   */
+  containsAllDataOf(other) {
+    for (const blockIndex of other.getBlockIndexes()) {
+      if (!this.hasBlock(blockIndex)) return false
+    }
+    return true
+  }
+
   /**
    * Checks if the VHD contains a specific block.
    * @param {number} index
@@ -205,11 +241,34 @@ export class RemoteDisk extends RandomAccessDisk {
   /**
    * Abstract
    * Deletes alias/disk/disks
+   * @param {Object} options
    */
-  async unlink() {
+  async unlink({ force = false } = {}) {
     throw new Error(`unlink must be implemented`)
   }
 
+  /**
+   * Checks the integrity of the disk's external reference (e.g. alias target).
+   * @param {Object} [opts]
+   * @param {boolean} [opts.remove]
+   * @param {Function} [opts.logWarn]
+   * @param {Function} [opts.logInfo]
+   * @returns {Promise<string | undefined>}
+   */
+  async clean(opts = {}) {
+    return undefined
+  }
+
+  /**
+   * Returns all file paths within dir that this disk claims.
+   *
+   * @param {string} dir
+   * @returns {Promise<string[]>}
+   */
+  async listAssociatedFiles(dir) {
+    throw new Error(`listAssociatedFiles must be implemented`)
+  }
+
   /**
    * @returns  {Promise<RemoteDisk>}
    */

@xen-orchestra/backup-archive/src/disks/RemoteDisk.test.mjs

@@ -972,6 +972,53 @@ describe('tests MergeRemoteDisk', { concurrency: 1 }, () => {
     assert.deepEqual(parent2.getBlockIndexes(), [0, 1, 2, 3], 'all blocks from the chain should be merged')
   })
 
+  test('mergeRemoteDisk resumes when state has currentBlock = -1', async () => {
+    // If the merge fails it will try to resume the merge at the lowest merging block - 1,
+    // if block 0 was being merged at this time, the state file currentBlock will be set to -1
+    // which is not a valid block id.
+
+    const parentVhdPath = `${basePath}/parent.vhd`
+    await generateVhd(parentVhdPath, { blocks: [1] })
+    const parent = new RemoteVhdDisk({ handler, path: parentVhdPath })
+    await parent.init({ force: false })
+
+    const childVhdPath = `${basePath}/child.vhd`
+    await generateVhd(childVhdPath, { blocks: [0] })
+    const child = new RemoteVhdDisk({ handler, path: childVhdPath })
+    await child.init({ force: false })
+
+    // Fail after asyncEach completes
+    parent.flushMetadata = async () => {
+      throw new Error('simulated interruption')
+    }
+
+    const mergeRemoteDisk = new MergeRemoteDisk(handler, { writeStateDelay: 0 })
+    let threw = false
+    try {
+      await mergeRemoteDisk.merge(parent, child)
+    } catch (err) {
+      threw = true
+      assert.equal(err.message, 'simulated interruption')
+    }
+    assert.ok(threw, 'first merge attempt should throw on simulated interruption')
+
+    const stateFileName = `.parent.vhd.merge.json`
+    assert.ok((await handler.list(basePath)).includes(stateFileName), 'state file should exist after failure')
+
+    // Resume with fresh instances
+    const parent2 = new RemoteVhdDisk({ handler, path: parentVhdPath })
+    await parent2.init({ force: true })
+    const child2 = new RemoteVhdDisk({ handler, path: childVhdPath })
+    await child2.init({ force: true })
+
+    const mergeRemoteDisk2 = new MergeRemoteDisk(handler, { removeUnused: false })
+    await mergeRemoteDisk2.merge(parent2, child2)
+
+    const indexes = parent2.getBlockIndexes()
+    assert.ok(indexes.includes(0), 'block 0 from child should be merged into parent')
+    assert.ok(indexes.includes(1), 'block 1 from parent should remain')
+  })
+
   test('mergeRemoteDisk merge a bigger child into a parent', async () => {
     const parentVhdPath = `${basePath}/parent.vhd`
     await generateVhd(parentVhdPath, { blocks: [0, 1] })

@xen-orchestra/backup-archive/src/disks/RemoteVhdDisk.mjs

@@ -3,9 +3,9 @@
 /**
  * @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
  * @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
- * @typedef {import('vhd-lib/_createFooterHeader').VhdFooter} VhdFooter
+ * @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
  * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
- * @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
+ * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
  *
 
  */
@@ -17,6 +17,7 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { stringify } from 'uuid'
 import { dirname, join } from 'node:path'
 import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
+import { normalize } from '@xen-orchestra/fs/path'
 
 export class RemoteVhdDisk extends RemoteDisk {
   /**
@@ -25,7 +26,7 @@ export class RemoteVhdDisk extends RemoteDisk {
   #path
 
   /**
-   * @type {FileAccessor}
+   * @type {RemoteHandlerAbstract}
    */
   #handler
 
@@ -56,7 +57,7 @@ export class RemoteVhdDisk extends RemoteDisk {
 
   /**
    * @param {Object} params
-   * @param {FileAccessor} params.handler
+   * @param {RemoteHandlerAbstract} params.handler
    * @param {string} params.path
    */
   constructor({ handler, path }) {
@@ -69,6 +70,7 @@ export class RemoteVhdDisk extends RemoteDisk {
   /**
    * @param {Object} [options]
    * @param {boolean} [options.force=false]
+   * @param {boolean} [options.ignoreBlockIndexes=false]
    * @returns {Promise<void>}
    */
   async init(options = {}) {
@@ -85,7 +87,9 @@ export class RemoteVhdDisk extends RemoteDisk {
         }
 
         this.#dispose = dispose
-        await this.#vhd.readBlockAllocationTable()
+        if (!options.ignoreBlockIndexes) {
+          await this.#vhd.readBlockAllocationTable()
+        }
         this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
       } catch (error) {
         await this.close()
@@ -172,6 +176,20 @@ export class RemoteVhdDisk extends RemoteDisk {
     return stringify(this.#vhd.header.parentUuid)
   }
 
+  /**
+   * Returns the parent path
+   *
+   * @returns {string}
+   */
+  getParentPath() {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call getParentPath of a RemoteVhdDisk before init`)
+    }
+
+    const parentPath = this.#vhd.header.parentUnicodeName
+    return normalize(join(dirname(this.#path), parentPath))
+  }
+
   /**
    * @returns {Promise<boolean>} canMergeConcurently
    */
@@ -218,7 +236,7 @@ export class RemoteVhdDisk extends RemoteDisk {
     }
 
     const parentPath = this.#vhd.header.parentUnicodeName
-    const fullParentPath = join(dirname(this.#path), parentPath)
+    const fullParentPath = normalize(join(dirname(this.#path), parentPath))
 
     if (!parentPath) {
       throw new Error(`disk ${this.#path} doesn't have parents`)
@@ -227,6 +245,13 @@ export class RemoteVhdDisk extends RemoteDisk {
     const parent = new RemoteVhdDisk({ handler: this.#handler, path: fullParentPath })
     return parent
   }
+  /**
+   * Parent of a RemoteVhdDisk is a RemoteVhdDisk
+   * @returns {Promise<RemoteVhdDisk>}
+   */
+  async openParent() {
+    return /** @type {Promise<RemoteVhdDisk>} */ (super.openParent())
+  }
 
   /**
    * Writes a full block into this VHD.
@@ -373,7 +398,7 @@ export class RemoteVhdDisk extends RemoteDisk {
   }
 
   /**
-   * @param {RemoteVhdDisk} childDisk
+   * @param {RemoteDisk} childDisk
    * @returns {Promise<void>}
    */
   async mergeMetadata(childDisk) {
@@ -416,10 +441,9 @@ export class RemoteVhdDisk extends RemoteDisk {
     if (isVhdAlias(newPath)) {
       const dataPath = await resolveVhdAlias(this.#handler, this.#path)
 
-      await this.#handler.unlink(this.#path)
       await this.#handler.unlink(newPath)
-
       await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
+      await this.#handler.unlink(this.#path)
 
       this.#path = newPath
     } else {
@@ -439,29 +463,57 @@ export class RemoteVhdDisk extends RemoteDisk {
 
   /**
    * Deletes disk
+   * @param {Object} options
    */
-  async unlink() {
+  async unlink({ force = false } = {}) {
     if (this.#vhd === undefined) {
-      throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
-    }
+      if (force) {
+        let resolved = this.#path
+        try {
+          resolved = await resolveVhdAlias(this.#handler, this.#path)
+        } catch (err) {
+          // broken vhd directory must be unlinkable
+          if (err.code !== 'EISDIR') {
+            throw err
+          }
+          // warn('Deleting directly a VhdDirectory', { this.#path, err })
+        }
+        try {
+          await this.#handler.unlink(resolved)
+        } catch (err) {
+          if (err.code === 'EISDIR') {
+            await this.#handler.rmtree(resolved)
+          } else {
+            throw err
+          }
+        }
 
-    await this.close()
+        // also delete the alias file
+        if (this.#path !== resolved) {
+          await this.#handler.unlink(this.#path)
+        }
+      } else {
+        throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
+      }
+    } else {
+      await this.close()
 
-    if (isVhdAlias(this.#path)) {
-      try {
-        await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
-      } catch (err) {
-        if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
-          await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
+      if (isVhdAlias(this.#path)) {
+        try {
+          await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
+        } catch (err) {
+          if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
+            await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
+          }
         }
       }
-    }
 
-    try {
-      await this.#handler.unlink(this.#path)
-    } catch (err) {
-      if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
-        await this.#handler.rmtree(this.#path).catch(() => {})
+      try {
+        await this.#handler.unlink(this.#path)
+      } catch (err) {
+        if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
+          await this.#handler.rmtree(this.#path).catch(() => {})
+        }
       }
     }
   }
@@ -477,4 +529,118 @@ export class RemoteVhdDisk extends RemoteDisk {
 
     return this.#vhd instanceof VhdDirectory
   }
+
+  /**
+   * Returns all file paths within dir that this disk claims.
+   * For a plain VHD or VHD directory: [path]
+   * For a VHD alias: [aliasPath, resolvedDataPath]
+   * If the alias target cannot be resolved, only the alias path is returned.
+   *
+   * @param {string} dir
+   * @returns {Promise<string[]>}
+   */
+  async listAssociatedFiles(dir) {
+    const prefix = normalize(dir.endsWith('/') ? dir : dir + '/')
+    const isInDir = /** @param {string} p */ p => p === dir || p.startsWith(prefix)
+
+    const files = []
+    if (isInDir(this.#path)) {
+      files.push(this.#path)
+    }
+
+    if (isVhdAlias(this.#path)) {
+      try {
+        const resolved = await resolveVhdAlias(this.#handler, this.#path)
+        if (isInDir(resolved)) {
+          files.push(resolved)
+        }
+      } catch {
+        // broken alias, no data file to claim
+      }
+    }
+
+    return files
+  }
+
+  /**
+   * Checks the integrity of this disk's alias reference.
+   * Only meaningful for alias files (.alias.vhd); no-op for plain VHDs.
+   * Returns the resolved target path when the alias is valid, undefined otherwise.
+   *
+   * @param {Object} [opts]
+   * @param {boolean} [opts.remove]
+   * @param {Function} [opts.logWarn]
+   * @param {Function} [opts.logInfo]
+   * @returns {Promise<string | undefined>}
+   */
+  async clean({ remove = false, logWarn = () => {}, logInfo = () => {} } = {}) {
+    if (!isVhdAlias(this.#path)) {
+      return undefined
+    }
+
+    let target
+    try {
+      target = await resolveVhdAlias(this.#handler, this.#path)
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        logWarn('missing target of alias', { alias: this.#path })
+        if (remove) {
+          logInfo('removing alias with missing target', { alias: this.#path })
+          await this.#handler.unlink(this.#path)
+        }
+        return undefined
+      }
+      if (err.code === 'EISDIR') {
+        logWarn('alias is a vhd directory', { alias: this.#path })
+        if (remove) {
+          logInfo('removing vhd directory named as alias', { alias: this.#path })
+          await VhdAbstract.unlink(this.#handler, this.#path)
+        }
+        return undefined
+      }
+      logWarn('unhandled error while checking alias', { alias: this.#path, err })
+      return undefined
+    }
+
+    if (target === '') {
+      logWarn('empty target for alias', { alias: this.#path })
+      if (remove) {
+        logInfo('removing alias with empty target', { alias: this.#path })
+        await this.#handler.unlink(this.#path)
+      }
+      return undefined
+    }
+
+    if (!isVhdAlias(target) && !target.endsWith('.vhd')) {
+      logWarn('alias references non VHD target', { alias: this.#path, target })
+      if (remove) {
+        logInfo('removing alias and non VHD target', { alias: this.#path, target })
+        await this.#handler.unlink(target)
+        await this.#handler.unlink(this.#path)
+      }
+      return undefined
+    }
+
+    try {
+      const { dispose } = await openVhd(this.#handler, target)
+      try {
+        await dispose()
+      } catch (_) {
+        // errors during dispose should not trigger deletion
+      }
+      return target
+    } catch (error) {
+      logWarn('missing or broken alias target', { alias: this.#path, target, error })
+      if (remove) {
+        try {
+          await VhdAbstract.unlink(this.#handler, this.#path)
+        } catch (/** @type {any} */ err) {
+          if (err.code !== 'ENOENT') {
+            logWarn('error deleting broken alias', { alias: this.#path, target, err })
+          }
+        }
+      }
+      return undefined
+    }
+  }
 }

@xen-orchestra/backup-archive/src/disks/RemoteVhdDiskChain.mjs

@@ -4,7 +4,6 @@
  * @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
  * @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
  * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
- * @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
  */
 
 import { RemoteDisk } from './RemoteDisk.mjs'
@@ -37,7 +36,6 @@ export class RemoteVhdDiskChain extends RemoteDisk {
 
   /**
    * @param {Object} params
-   * @param {FileAccessor} params.handler
    * @param {RemoteVhdDisk[]} params.disks
    */
   constructor({ disks }) {
@@ -248,7 +246,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
   }
 
   /**
-   * @param {RemoteVhdDisk} childDisk
+   * @param {RemoteDisk} childDisk
    * @returns {Promise<void>}
    */
   async flushMetadata(childDisk) {
@@ -256,7 +254,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
   }
 
   /**
-   * @param {RemoteVhdDisk} childDisk
+   * @param {RemoteDisk} childDisk
    * @returns {Promise<void>}
    */
   mergeMetadata(childDisk) {
@@ -282,10 +280,11 @@ export class RemoteVhdDiskChain extends RemoteDisk {
 
   /**
    * Deletes all the disks
+   * @param {Object} options
    */
-  async unlink() {
+  async unlink({ force = false } = {}) {
     for (const disk of this.#disks) {
-      await disk.unlink()
+      await disk.unlink({ force })
     }
   }
 
@@ -301,4 +300,24 @@ export class RemoteVhdDiskChain extends RemoteDisk {
     }
     return true
   }
+
+  /**
+   * Checks the integrity of the disk's external reference (e.g. alias target).
+   * @param {Object} [opts]
+   * @param {boolean} [opts.remove]
+   * @param {Function} [opts.logWarn]
+   * @param {Function} [opts.logInfo]
+   * @returns {Promise<string | undefined>}
+   */
+  async clean(opts = {}) {
+    throw new Error(`Can't clean a disk chain`)
+  }
+
+  /**
+   *
+   * @returns {RemoteVhdDisk|undefined}
+   */
+  getRootDisk() {
+    return this.#disks[0]
+  }
 }

@xen-orchestra/backup-archive/src/disks/index.mjs

@@ -0,0 +1,104 @@
+import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
+
+export { RemoteDisk } from './RemoteDisk.mjs'
+export { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
+export { openDiskChain } from './openDiskChain.mjs'
+export { MergeRemoteDisk } from './MergeRemoteDisk.mjs'
+
+const DISK_EXTENSIONS = ['.vhd']
+
+/**
+ * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
+ * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
+ * @typedef {import('./MergeRemoteDisk.mjs').MergeState} MergeState
+ */
+
+export function isDisk(path) {
+  return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
+}
+
+/**
+ *
+ * @param {Object} params
+ * @param {RemoteHandlerAbstract} params.handler
+ * @param {string} params.path
+ * @param {boolean} [params.force]
+ * @param {boolean} [params.ignoreBlockIndexes]
+ * @returns {Promise<Disposable<RemoteDisk>>}
+ */
+export async function openDisposableDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
+  const disk = new RemoteVhdDisk({ handler, path })
+  if (ignoreBlockIndexes) {
+    // Best-effort init: return the disk even if init fails so that callers
+    // can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
+    await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
+  } else {
+    await disk.init({ force })
+  }
+  return {
+    value: disk,
+    dispose: () => disk.close(),
+  }
+}
+
+/**
+ * @param {Object} params
+ * @param {RemoteHandlerAbstract} params.handler
+ * @param {string} params.path
+ * @param {boolean} [params.force]
+ * @param {boolean} [params.ignoreBlockIndexes]
+ * @returns {Promise<RemoteDisk>}
+ */
+export async function openDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
+  const disk = new RemoteVhdDisk({ handler, path })
+  if (ignoreBlockIndexes) {
+    // Best-effort init: return the disk even if init fails so that callers
+    // can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
+    await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
+  } else {
+    await disk.init({ force })
+  }
+  return disk
+}
+
+/**
+ * Deletes VDI directories under `vmDir/vdis/<jobId>/<vdiId>/` that are not in coveredDirs.
+ * Used to clean up disk directories that no backup metadata references at all.
+ *
+ * @param {RemoteHandlerAbstract} handler
+ * @param {string} vmDir  - root of the VM backup directory (xo-vm-backups/<vmUuid>)
+ * @param {Set<string>} coveredDirs - normalized vdiDir paths already managed by a lineage
+ * @param {{ remove?: boolean, logWarn?: Function, logInfo?: Function }} opts
+ */
+export async function cleanOrphanDiskDirs(
+  handler,
+  vmDir,
+  coveredDirs,
+  { remove = false, logWarn = () => {}, logInfo = () => {} } = {}
+) {
+  const vdisPath = `${vmDir}/vdis`
+  let jobDirs
+  try {
+    jobDirs = await handler.list(vdisPath, { prependDir: true })
+  } catch {
+    return
+  }
+
+  for (const jobDir of jobDirs) {
+    let vdiDirs
+    try {
+      vdiDirs = await handler.list(jobDir, { prependDir: true })
+    } catch {
+      continue
+    }
+    for (const vdiDir of vdiDirs) {
+      if (!coveredDirs.has(vdiDir)) {
+        // TODO: readd logWarn when rmtree() is added below
+        if (remove) {
+          logInfo('you could delete orphan VDI directory', { path: vdiDir })
+          // TODO: Readd rmtree() when codebase is ready
+        }
+      }
+    }
+  }
+}

@xen-orchestra/backup-archive/src/disks/openDiskChain.mjs

@@ -0,0 +1,57 @@
+// @ts-check
+/**
+ *
+ * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
+ * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
+ */
+import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
+import { defer } from 'golike-defer'
+import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
+
+/**
+ * @param {any} $defer
+ * @param {Object} params
+ * @param {RemoteHandlerAbstract} params.handler
+ * @param {string} params.path
+ * @param {string | undefined} params.until
+ * @param {boolean} [params.force]
+ * @param {boolean} [params.ignoreBlockIndexes] - skip reading the block allocation table of each disk (cheaper, but blocks can't be read)
+ * @returns {Promise<RemoteDisk>}
+ */
+async function _openDiskChain($defer, { handler, path, until, force = false, ignoreBlockIndexes = false }) {
+  /**
+   * @type {RemoteVhdDisk}
+   */
+  let disk
+  /**
+   * @type {Array<RemoteVhdDisk>}
+   */
+  const disks = []
+  $defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
+  disk = new RemoteVhdDisk({ handler, path })
+
+  await disk.init({ force, ignoreBlockIndexes })
+  disks.push(disk)
+  let foundRootDisk = until === undefined
+  while (disk.isDifferencing()) {
+    if (disk.getParentPath() === until) {
+      foundRootDisk = true
+      break
+    }
+    // Instantiate + init the parent directly (instead of openParent()) so the
+    // block allocation table read can be skipped when ignoreBlockIndexes is set.
+    disk = /** @type {RemoteVhdDisk} */ (disk.instantiateParent())
+    await disk.init({ force, ignoreBlockIndexes })
+    disks.unshift(disk)
+  }
+  if (!foundRootDisk) {
+    throw new Error(`trying to open chain from ${path} to ${until}, root disk not found`)
+  }
+  // the root disk
+  return new RemoteVhdDiskChain({ disks })
+}
+
+/**
+ * @type {(params: { handler: RemoteHandlerAbstract, path: string, until?: string, force?: boolean, ignoreBlockIndexes?: boolean }) => Promise<RemoteDisk>}
+ */
+export const openDiskChain = defer(_openDiskChain)

@xen-orchestra/backup-archive/src/index.mts

@@ -0,0 +1 @@
+export { VmBackupDirectory } from './VmBackupDirectory.mjs'

@xen-orchestra/backup-archive/src/tests.fixtures.mjs

@@ -0,0 +1,92 @@
+// a valid footer of a 2
+export const VHDFOOTER = {
+  cookie: 'conectix',
+  features: 2,
+  fileFormatVersion: 65536,
+  dataOffset: 512,
+  timestamp: 0,
+  creatorApplication: 'caml',
+  creatorVersion: 1,
+  creatorHostOs: 0,
+  originalSize: 53687091200,
+  currentSize: 53687091200,
+  diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
+  diskType: 3,
+  checksum: 4294962945,
+  uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
+  saved: '',
+  hidden: '',
+  reserved: '',
+}
+export const VHDHEADER = {
+  cookie: 'cxsparse',
+  dataOffset: undefined,
+  tableOffset: 2048,
+  headerVersion: 65536,
+  maxTableEntries: 25600,
+  blockSize: 2097152,
+  checksum: 4294964241,
+  parentUuid: null,
+  parentTimestamp: 0,
+  reserved1: 0,
+  parentUnicodeName: '',
+  parentLocatorEntry: [
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+  ],
+  reserved2: '',
+}

@xen-orchestra/backup-archive/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist",
+    "declaration": true,
+    "strict": true,
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
+    "target": "ES2022",
+    "rootDir": "./src",
+    "skipLibCheck": true,
+    "noImplicitAny": false,
+    "allowJs": true,
+    "useUnknownInCatchVariables": false,
+    "types": ["node"] // Only use types from Node.js and packages that are explicitly imported
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

@xen-orchestra/backups-cli/commands/clean-vms.mjs

@@ -19,7 +19,7 @@ async function cleanFiles(vmDirs, { fix, merge, remove }) {
         try {
           nbStarted++
           await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
-            fixMetadata: fix,
+            fix,
             remove,
             merge,
             logInfo: (...args) => console.log(...args),
@@ -50,7 +50,7 @@ async function cleanRemote(url, { fix, merge, remove }) {
         try {
           nbStarted++
           await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
-            fixMetadata: fix,
+            fix,
             remove,
             merge,
             logInfo: (...args) => console.log(...args),

@xen-orchestra/backups-cli/package.json

@@ -8,8 +8,8 @@
   "dependencies": {
     "@vates/async-each": "^1.0.3",
     "@xen-orchestra/async-map": "^0.1.3",
-    "@xen-orchestra/backups": "^0.72.0",
-    "@xen-orchestra/fs": "^4.8.0",
+    "@xen-orchestra/backups": "^0.73.10",
+    "@xen-orchestra/fs": "^4.9.3",
     "filenamify": "^6.0.0",
     "getopts": "^2.2.5",
     "lodash": "^4.18.0",
@@ -28,7 +28,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.1.11",
+  "version": "1.1.13",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -10,7 +10,7 @@ import { dirname, join } from 'node:path'
 import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
 import { NegativeDisk } from '@xen-orchestra/disk-transform'
-import { openDiskChain } from './disks/openDiskChain.mjs'
+import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
 import { resetVmOtherConfig } from './_otherConfig.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -1,38 +1,26 @@
 import { asyncEach } from '@vates/async-each'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
-import { compose } from '@vates/compose'
 import { createLogger } from '@xen-orchestra/log'
 import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
 import { decorateMethodsWith } from '@vates/decorate-with'
-import { deduped } from '@vates/disposable/deduped.js'
 import { dirname, join, resolve } from 'node:path'
-import { execFile } from 'child_process'
-import { mount } from '@vates/fuse-vhd'
-import { readdir, lstat } from 'node:fs/promises'
 import { synchronized } from 'decorator-synchronized'
-import { ZipFile } from 'yazl'
 import Disposable from 'promise-toolbox/Disposable'
 import fromCallback from 'promise-toolbox/fromCallback'
-import fromEvent from 'promise-toolbox/fromEvent'
 import groupBy from 'lodash/groupBy.js'
-import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
 import reduce from 'lodash/reduce.js'
-import * as tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'
-import { cleanVm } from './_cleanVm.mjs'
+import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
+import { fileRestoreDecorators, fileRestoreMethods } from './_fileRestore.mjs'
 import { formatFilenameDate } from './_filenameDate.mjs'
-import { getTmpDir } from './_getTmpDir.mjs'
 import { isMetadataFile } from './_backupType.mjs'
 import { isValidXva } from './_isValidXva.mjs'
-import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
-import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
-import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
-import { openDiskChain } from './disks/openDiskChain.mjs'
+import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
 import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
 import { ReadAhead } from '@xen-orchestra/disk-transform'
 
@@ -49,25 +37,6 @@ export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 const noop = Function.prototype
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
-const makeRelative = path => resolve('/', path).slice(1)
-const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
-
-async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
-  for (const relativePath of relativePaths) {
-    const realPath = join(realBasePath, relativePath)
-    const virtualPath = join(virtualBasePath, relativePath)
-
-    const stats = await lstat(realPath)
-    const { mode, mtime } = stats
-    const opts = { mode, mtime }
-    if (stats.isDirectory()) {
-      zip.addEmptyDirectory(virtualPath, opts)
-      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
-    } else if (stats.isFile()) {
-      zip.addFile(realPath, virtualPath, opts)
-    }
-  }
-}
 
 const createSafeReaddir = (handler, methodName) => (path, options) =>
   handler.list(path, options).catch(error => {
@@ -77,11 +46,6 @@ const createSafeReaddir = (handler, methodName) => (path, options) =>
     return []
   })
 
-const debounceResourceFactory = factory =>
-  function () {
-    return this._debounceResource(factory.apply(this, arguments))
-  }
-
 export class RemoteAdapter {
   constructor(
     handler,
@@ -99,112 +63,6 @@ export class RemoteAdapter {
     return this._handler
   }
 
-  async _findPartition(devicePath, partitionId) {
-    const partitions = await listPartitions(devicePath)
-    const partition = partitions.find(_ => _.id === partitionId)
-    if (partition === undefined) {
-      throw new Error(`partition ${partitionId} not found`)
-    }
-    return partition
-  }
-
-  async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
-    yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
-
-    debug('activate LVM volume group', { vgName })
-    await fromCallback(execFile, 'vgchange', ['-ay', vgName])
-    try {
-      debug('get LVM volume group name and path', { vgName })
-      yield lvs(['lv_name', 'lv_path'], vgName)
-    } finally {
-      debug('deactivate LVM volume group', { vgName })
-      await fromCallback(execFile, 'vgchange', ['-an', vgName])
-    }
-  }
-
-  async *_getLvmPhysicalVolume(devicePath, partition) {
-    const args = []
-    if (partition !== undefined) {
-      args.push('-o', partition.start * 512, '--sizelimit', partition.size)
-    }
-    args.push('--show', '-f', devicePath)
-
-    debug('attach loop device', { devicePath, partition })
-    const path = (await fromCallback(execFile, 'losetup', args)).trim()
-    try {
-      debug('list LVM physical volume', { path })
-      await fromCallback(execFile, 'pvscan', ['--cache', path])
-
-      yield path
-    } finally {
-      try {
-        const vgNames = await pvs('vg_name', path)
-
-        debug('deactivate LVM volume groups', { vgNames })
-        await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
-      } finally {
-        debug('detach loop device', { path })
-        await fromCallback(execFile, 'losetup', ['-d', path])
-      }
-    }
-  }
-
-  async *_getPartition(devicePath, partition) {
-    // the norecovery option is necessary because if the partition is dirty,
-    // mount will try to fix it which is impossible if because the device is read-only
-    const options = ['loop', 'ro', 'norecovery']
-
-    if (partition !== undefined) {
-      const { size, start } = partition
-      options.push(`sizelimit=${size}`)
-      if (start !== undefined) {
-        options.push(`offset=${start * 512}`)
-      }
-    }
-
-    const path = yield getTmpDir()
-    const mount = options => {
-      debug('mount device', { devicePath, mountPath: path })
-      return fromCallback(execFile, 'mount', [
-        `--options=${options.join(',')}`,
-        `--source=${devicePath}`,
-        `--target=${path}`,
-      ])
-    }
-
-    // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
-    // another fs, try without
-    try {
-      await mount([...options, 'norecovery'])
-    } catch (error) {
-      await mount(options)
-    }
-    try {
-      yield path
-    } finally {
-      debug('umount device', { devicePath, mountPath: path })
-      await fromCallback(execFile, 'umount', ['--lazy', path])
-    }
-  }
-
-  _listLvmLogicalVolumes(devicePath, partition, results = []) {
-    return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => {
-      const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path)
-      const partitionId = partition !== undefined ? partition.id : ''
-      lvs.forEach((lv, i) => {
-        const name = lv.lv_name
-        if (name !== '') {
-          results.push({
-            id: `${partitionId}/${lv.vg_name}/${name}`,
-            name,
-            size: lv.lv_size,
-          })
-        }
-      })
-      return results
-    })
-  }
-
   // check if we will be allowed to merge a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
@@ -222,34 +80,6 @@ export class RemoteAdapter {
     })
   }
 
-  fetchPartitionFiles(diskId, partitionId, paths, format) {
-    const { promise, reject, resolve } = pDefer()
-    Disposable.use(
-      async function* () {
-        const path = yield this.getPartition(diskId, partitionId)
-        let outputStream
-
-        if (format === 'tgz') {
-          outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
-        } else if (format === 'zip') {
-          const zip = new ZipFile()
-          await addZipEntries(zip, path, '', paths.map(makeRelative))
-          zip.end()
-          ;({ outputStream } = zip)
-        } else {
-          throw new Error('unsupported format ' + format)
-        }
-
-        resolve(outputStream)
-        await fromEvent(outputStream, 'end')
-      }.bind(this)
-    ).catch(error => {
-      warn(error)
-      reject(error)
-    })
-    return promise
-  }
-
   async #removeVmBackupsFromCache(backups) {
     await asyncEach(
       Object.entries(
@@ -301,7 +131,20 @@ export class RemoteAdapter {
   async deleteFullVmBackups(backups) {
     const handler = this._handler
     await asyncMapSettled(backups, ({ _filename, xva }) =>
-      Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))])
+      Promise.all([
+        handler.unlink(_filename).catch(error => {
+          warn('error while removing full vm backup metadata', { error, filename: _filename })
+          if (error.code !== 'ENOENT') throw error
+        }),
+        handler.unlink(resolveRelativeFromFile(_filename, xva)).catch(error => {
+          warn('error while removing full vm backup file', { error, filename: _filename })
+          if (error.code !== 'ENOENT') throw error
+        }),
+        handler.unlink(resolveRelativeFromFile(_filename, `${xva}.checksum`)).catch(error => {
+          // checksum can be missing , it's not an issue
+          if (error.code !== 'ENOENT') throw error
+        }),
+      ])
     )
 
     await this.#removeVmBackupsFromCache(backups)
@@ -312,18 +155,46 @@ export class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const metadata = await asyncMap(files, file => this.readVmBackupMetadata(file))
-    const { delta, full, ...others } = groupBy(metadata, 'mode')
+    const metadataOrNull = await asyncMap(files, async file => {
+      try {
+        return await this.readVmBackupMetadata(file)
+      } catch (error) {
+        if (error.code === 'ENOENT') {
+          // File was already removed (e.g. by coalescing); clean the stale cache entry
+          warn('backup metadata not found, removing stale cache entry', { file })
+          return null
+        }
+        throw error
+      }
+    })
+
+    const presentMetadata = []
+    const missingFiles = []
+    for (let i = 0; i < files.length; i++) {
+      if (metadataOrNull[i] === null) {
+        missingFiles.push({ _filename: files[i] })
+      } else {
+        presentMetadata.push(metadataOrNull[i])
+      }
+    }
+
+    const { delta, full, ...others } = groupBy(presentMetadata, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
       throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
     }
-
-    await Promise.all([
-      delta !== undefined && this.deleteDeltaVmBackups(delta),
-      full !== undefined && this.deleteFullVmBackups(full),
-    ])
+    const promises = []
+    if (delta !== undefined) {
+      promises.push(this.deleteDeltaVmBackups(delta))
+    }
+    if (full !== undefined) {
+      promises.push(this.deleteFullVmBackups(full))
+    }
+    if (missingFiles.length) {
+      promises.push(this.#removeVmBackupsFromCache(missingFiles))
+    }
+    await Promise.all(promises)
 
     await asyncMap(new Set(files.map(file => dirname(file))), dir =>
       // - don't merge in main process, unused VHDs will be merged in the next backup run
@@ -346,76 +217,6 @@ export class RemoteAdapter {
     return this.useVhdDirectory()
   }
 
-  async *#getDiskLegacy(diskId) {
-    const RE_VHDI = /^vhdi(\d+)$/
-    const handler = this._handler
-
-    const diskPath = handler.getFilePath('/' + diskId)
-    const mountDir = yield getTmpDir()
-
-    debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
-    await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
-    try {
-      let max = 0
-      let maxEntry
-      const entries = await readdir(mountDir)
-      entries.forEach(entry => {
-        const matches = RE_VHDI.exec(entry)
-        if (matches !== null) {
-          const value = +matches[1]
-          if (value > max) {
-            max = value
-            maxEntry = entry
-          }
-        }
-      })
-      if (max === 0) {
-        throw new Error('no disks found')
-      }
-
-      yield `${mountDir}/${maxEntry}`
-    } finally {
-      debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
-      await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
-    }
-  }
-
-  async *getDisk(diskId) {
-    if (this._useGetDiskLegacy) {
-      yield* this.#getDiskLegacy(diskId)
-      return
-    }
-    const handler = this._handler
-    // this is a disposable
-    const mountDir = yield getTmpDir()
-    // this is also a disposable
-    yield mount(handler, diskId, mountDir)
-    // this will yield disk path to caller
-    yield `${mountDir}/vhd0`
-  }
-
-  // partitionId values:
-  //
-  // - undefined: raw disk
-  // - `<partitionId>`: partitioned disk
-  // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
-  // - `/<vgName>/lvName>`: LVM on a raw disk
-  async *getPartition(diskId, partitionId) {
-    const devicePath = yield this.getDisk(diskId)
-    if (partitionId === undefined) {
-      return yield this._getPartition(devicePath)
-    }
-
-    const isLvmPartition = partitionId.includes('/')
-    if (isLvmPartition) {
-      const [pvId, vgName, lvName] = partitionId.split('/')
-      const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
-      return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
-    }
-
-    return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
-  }
-
   // if we use alias on this remote, we have to name the file alias.vhd
   getVhdFileName(baseName) {
     if (this.#useAlias()) {
@@ -456,56 +257,6 @@ export class RemoteAdapter {
     return backups
   }
 
-  listPartitionFiles(diskId, partitionId, path) {
-    return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
-      path = resolveSubpath(rootPath, path)
-      const entriesMap = {}
-      await asyncEach(
-        await readdir(path),
-        async name => {
-          try {
-            const stats = await lstat(`${path}/${name}`)
-            if (stats.isDirectory()) {
-              entriesMap[name + '/'] = {}
-            } else if (stats.isFile()) {
-              entriesMap[name] = {}
-            }
-          } catch (error) {
-            if (error == null || error.code !== 'ENOENT') {
-              throw error
-            }
-          }
-        },
-        { concurrency: 1 }
-      )
-
-      return entriesMap
-    })
-  }
-
-  listPartitions(diskId) {
-    return Disposable.use(this.getDisk(diskId), async devicePath => {
-      const partitions = await listPartitions(devicePath)
-
-      if (partitions.length === 0) {
-        try {
-          // handle potential raw LVM physical volume
-          return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
-        } catch (error) {
-          return []
-        }
-      }
-
-      const results = []
-      await asyncMapSettled(partitions, partition =>
-        partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
-          ? this._listLvmLogicalVolumes(devicePath, partition, results)
-          : results.push(partition)
-      )
-      return results
-    })
-  }
-
   async listPoolMetadataBackups() {
     const handler = this._handler
     const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
@@ -564,6 +315,11 @@ export class RemoteAdapter {
       fn(cache)
 
       await this._writeCache(path, cache)
+    } else {
+      const regenerated = await this.#getCacheableDataListVmBackups(dirname(path))
+      if (regenerated !== undefined) {
+        await this._writeCache(path, regenerated)
+      }
     }
   }
 
@@ -632,8 +388,7 @@ export class RemoteAdapter {
       return
     }
 
-    // detached async action, will not reject
-    this._writeCache(path, backups)
+    await this._writeCache(path, backups)
 
     return backups
   }
@@ -698,11 +453,11 @@ export class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
+  async writeVhd(path, disk, { validator = noop, writeBlockConcurrency, uuid, parentUuid, parentPath } = {}) {
     const handler = this._handler
 
     if (this.useVhdDirectory()) {
-      await writeToVhdDirectory({
+      return await writeToVhdDirectory({
         disk,
         target: {
           handler,
@@ -710,12 +465,21 @@ export class RemoteAdapter {
           concurrency: writeBlockConcurrency,
           validator,
           compression: 'brotli',
+          uuid,
+          parentUuid,
+          parentPath,
         },
       })
     } else {
-      const stream = await toVhdStream(disk)
-      await this.outputStream(path, stream, { validator, checksum: false })
+      const stream = await toVhdStream(disk, { uuid, parentUuid, parentPath })
+      const size = await this.outputStream(path, stream, {
+        validator,
+        // no checksum for VHDs, because they will be invalidated by
+        // merges and chains
+        checksum: false,
+      })
       await validator(path)
+      return size
     }
   }
 
@@ -878,36 +642,21 @@ export class RemoteAdapter {
 }
 
 Object.assign(RemoteAdapter.prototype, {
-  cleanVm(vmDir, { lock = true } = {}) {
+  cleanVm(vmBackupPath, opts = {}) {
+    const { lock = true, ...cleanOpts } = opts
     if (lock) {
-      return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
+      return Disposable.use(this._handler.lock(vmBackupPath), () => {
+        return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
+      })
     } else {
-      return cleanVm.apply(this, arguments)
+      return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
     }
   },
   isValidXva,
 })
 
-decorateMethodsWith(RemoteAdapter, {
-  _getLvmLogicalVolumes: compose([
-    Disposable.factory,
-    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
-    debounceResourceFactory,
-  ]),
+// File-level-restore methods live in ./_fileRestore.mjs; mix them onto the prototype
+// before decorating so decorateMethodsWith can wrap them.
+Object.assign(RemoteAdapter.prototype, fileRestoreMethods)
 
-  _getLvmPhysicalVolume: compose([
-    Disposable.factory,
-    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
-    debounceResourceFactory,
-  ]),
-
-  _getPartition: compose([
-    Disposable.factory,
-    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
-    debounceResourceFactory,
-  ]),
-
-  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
-
-  getPartition: Disposable.factory,
-})
+decorateMethodsWith(RemoteAdapter, fileRestoreDecorators)

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -9,7 +9,6 @@ import { pFromCallback } from 'promise-toolbox'
 import { RemoteAdapter } from './RemoteAdapter.mjs'
 import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
 import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
-import { checkAliases } from './_cleanVm.mjs'
 import { dirname, basename } from 'node:path'
 import { rimraf } from 'rimraf'
 
@@ -82,13 +81,17 @@ test('It remove broken vhd', async () => {
   // todo also tests a directory and an alias
 
   await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/notReallyAVhd.vhd`] })
+  )
   assert.equal((await handler.list(basePath)).length, 1)
   let logged = ''
   const logInfo = message => {
     logged += message
   }
   await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
-  assert.equal(logged, `VHD check error`)
+  assert.ok(logged.includes('disk check error'), `expected 'disk check error' in: ${logged}`)
   // not removed
   assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
   // really remove it
@@ -115,10 +118,10 @@ test('it remove vhd with missing or multiple ancestors', async () => {
     },
   })
   await handler.writeFile(
-    `metadata.json`,
+    `${rootPath}/metadata.json`,
     JSON.stringify({
       mode: 'delta',
-      vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
+      vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
     }),
     { flags: 'w' }
   )
@@ -129,13 +132,13 @@ test('it remove vhd with missing or multiple ancestors', async () => {
   }
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
 
-  const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
-  assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
+  const detectedMissingParent = logged.match(/parent disk is missing/g) || []
+  assert.equal(detectedMissingParent.length, 1) // only one vhd should have been deleted
 
   // we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
 })
 
-test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
+test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
   // create a metadata file marking child and orphan as ok
   await handler.writeFile(
     `${rootPath}/metadata.json`,
@@ -167,7 +170,7 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
     logged += message + '\n'
   }
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
-  let matched = logged.match(/deleting unused VHD/g) || []
+  let matched = logged.match(/deleting unused disk/g) || []
   assert.equal(matched.length, 1) // only one vhd should have been deleted
 
   // a missing vhd cause clean to remove all vhds
@@ -176,9 +179,9 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
     JSON.stringify({
       mode: 'delta',
       vhds: [
-        `deleted.vhd`, // in metadata but not in vhds
-        `orphan.vhd`,
-        `child.vhd`,
+        `${relativePath}/deleted.vhd`, // in metadata but missing from disk
+        `${relativePath}/orphan.vhd`,
+        `${relativePath}/child.vhd`,
         // abandoned.vhd is not here anymore
       ],
     }),
@@ -186,7 +189,7 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
   )
   logged = ''
   await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
-  matched = logged.match(/deleting unused VHD/g) || []
+  matched = logged.match(/deleting unused disk/g) || []
   assert.equal(matched.length, 2) // all vhds (orphan and  child  ) should have been deleted
 })
 
@@ -233,7 +236,7 @@ test('it merges delta of non destroyed chain', async () => {
   const result = await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
   assert.ok(result.size > 0, `no merged size`)
   const [merging] = logged
-  assert.equal(merging, `merging VHD chain`)
+  assert.equal(merging, `Disk chain needs merging`)
 
   const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
   // size should be the size of children + grand children after the merge
@@ -451,8 +454,8 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
         await adapter.cleanVm(rootPath, { remove: true, merge: true, logWarn: () => {}, lock: false })
 
         const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
-        // size should be the size of children + grand children + clean after the merge
-        assert.deepEqual(metadata.size, 4404224)
+        // size is the sum of the merged chains' final on-disk sizes:
+        assert.deepEqual(metadata.size, 6501888)
 
         // broken vhd, non referenced, abandoned should be deleted ( alias and data)
         // ancestor and child should be merged
@@ -481,32 +484,12 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
 })
 test('it cleans orphan merge states ', async () => {
   await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
+  await handler.writeFile(
+    `${rootPath}/metadata.json`,
+    JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
+  )
 
   await adapter.cleanVm(rootPath, { remove: true, logWarn: () => {}, lock: false })
 
   assert.deepEqual(await handler.list(basePath), [])
 })
-
-test('check Aliases should work alone', async () => {
-  await handler.mkdir('vhds')
-  await handler.mkdir('vhds/data')
-  await generateVhd(`vhds/data/ok.vhd`)
-  await VhdAbstract.createAlias(handler, 'vhds/ok.alias.vhd', 'vhds/data/ok.vhd')
-
-  await VhdAbstract.createAlias(handler, 'vhds/missingData.alias.vhd', 'vhds/data/nonexistent.vhd')
-
-  await generateVhd(`vhds/data/missingalias.vhd`)
-
-  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', {
-    remove: true,
-    handler,
-    logWarn: () => {},
-  })
-
-  // only ok have survived
-  const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
-  assert.equal(alias.length, 1)
-
-  const data = await handler.list('vhds/data')
-  assert.equal(data.length, 1)
-})

@xen-orchestra/backups/_cleanVm.mjs

@@ -1,628 +0,0 @@
-import * as UUID from 'uuid'
-import { asyncMap } from '@xen-orchestra/async-map'
-import { Constants, openVhd, VhdAbstract } from 'vhd-lib'
-import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
-import { basename, dirname, resolve } from 'node:path'
-import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
-import { limitConcurrency } from 'limit-concurrency-decorator'
-import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
-import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
-import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
-
-import { Disposable } from 'promise-toolbox'
-import { Task } from '@vates/task'
-import handlerPath from '@xen-orchestra/fs/path'
-
-const { DISK_TYPES } = Constants
-
-// chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
-  logInfo(`merging VHD chain`, { chain })
-
-  let done, total
-  const handle = setInterval(() => {
-    if (done !== undefined) {
-      logInfo('merge in progress', {
-        done,
-        parent: chain[0],
-        progress: Math.round((100 * done) / total),
-        total,
-      })
-    }
-  }, 10e3)
-  try {
-    const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
-
-    const childDisks = []
-    for (const path of chain) {
-      childDisks.push(new RemoteVhdDisk({ handler, path }))
-    }
-    const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
-
-    const mergeRemoteDisk = new MergeRemoteDisk(handler, {
-      logInfo,
-      mergeBlockConcurrency,
-      onProgress({ done: d, total: t }) {
-        done = d
-        total = t
-      },
-      removeUnused: remove,
-    })
-
-    const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
-    await parentDisk.init({ force: isResumingMerge })
-    await childDiskChain.init({ force: isResumingMerge })
-
-    const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
-
-    return result
-  } finally {
-    clearInterval(handle)
-  }
-}
-
-const noop = Function.prototype
-
-const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
-
-const listVhds = async (handler, vmDir, logWarn) => {
-  const vhds = new Set()
-  const aliases = {}
-  const interruptedVhds = new Map()
-  const checksums = new Set()
-
-  await asyncMap(
-    await handler.list(`${vmDir}/vdis`, {
-      ignoreMissing: true,
-      prependDir: true,
-    }),
-    async jobDir =>
-      asyncMap(
-        await handler.list(jobDir, {
-          prependDir: true,
-        }),
-        async vdiDir => {
-          const list = await handler.list(vdiDir, {
-            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
-          })
-          aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
-
-          await asyncMap(list, async file => {
-            if (isVhdSumFile(file)) {
-              checksums.add(`${vdiDir}/${file}`)
-              return
-            }
-            const res = INTERRUPTED_VHDS_REG.exec(file)
-            if (res === null) {
-              vhds.add(`${vdiDir}/${file}`)
-            } else {
-              try {
-                const mergeState = JSON.parse(await handler.readFile(`${vdiDir}/${file}`))
-                interruptedVhds.set(`${vdiDir}/${res[1]}`, {
-                  statePath: `${vdiDir}/${file}`,
-                  chain: mergeState.chain,
-                })
-              } catch (error) {
-                // fall back to a non resuming merge
-                vhds.add(`${vdiDir}/${file}`)
-                logWarn('failed to read existing merge state', { path: file, error })
-              }
-            }
-          })
-        }
-      )
-  )
-
-  return { vhds, interruptedVhds, aliases, checksums }
-}
-
-export async function checkAliases(
-  aliasPaths,
-  targetDataRepository,
-  { handler, logInfo = noop, logWarn = console.warn, remove = false }
-) {
-  const aliasFound = []
-  for (const alias of aliasPaths) {
-    let target
-    try {
-      target = await resolveVhdAlias(handler, alias)
-    } catch (err) {
-      if (err.code === 'ENOENT') {
-        logWarn('missing target of alias', { alias })
-        if (remove) {
-          logInfo('removing alias and non VHD target', { alias, target })
-          await handler.unlink(alias)
-        }
-        continue
-      }
-      if (err.code === 'EISDIR') {
-        logWarn('alias is a vhd directory', { alias })
-        if (remove) {
-          logInfo('removing vhd directory named as alias', { alias, target })
-          await VhdAbstract.unlink(handler, alias)
-        }
-        continue
-      }
-      logWarn('unhandled error while checking alias', { alias, err })
-      continue
-    }
-
-    if (target === '') {
-      logWarn('empty target for alias ', { alias })
-      if (remove) {
-        logInfo('removing alias and non VHD target', { alias, target })
-        await handler.unlink(alias)
-      }
-      continue
-    }
-
-    if (!isVhdFile(target)) {
-      logWarn('alias references non VHD target', { alias, target })
-      if (remove) {
-        logInfo('removing alias and non VHD target', { alias, target })
-        await handler.unlink(target)
-        await handler.unlink(alias)
-      }
-      continue
-    }
-
-    try {
-      const { dispose } = await openVhd(handler, target)
-      try {
-        await dispose()
-      } catch (e) {
-        // error during dispose should not trigger a deletion
-      }
-    } catch (error) {
-      logWarn('missing or broken alias target', { alias, target, error })
-      if (remove) {
-        try {
-          await VhdAbstract.unlink(handler, alias)
-        } catch (error) {
-          if (error.code !== 'ENOENT') {
-            logWarn('error deleting alias target', { alias, target, error })
-          }
-        }
-      }
-      continue
-    }
-
-    aliasFound.push(resolve('/', target))
-  }
-
-  const vhds = await handler.list(targetDataRepository, {
-    ignoreMissing: true,
-    prependDir: true,
-  })
-
-  await asyncMap(vhds, async path => {
-    if (!aliasFound.includes(path)) {
-      logWarn('no alias references VHD', { path })
-      if (remove) {
-        logInfo('deleting unused VHD', { path })
-        await VhdAbstract.unlink(handler, path)
-      }
-    }
-  })
-}
-
-const defaultMergeLimiter = limitConcurrency(1)
-
-export async function cleanVm(
-  vmDir,
-  {
-    fixMetadata,
-    remove = false,
-    removeTmp = remove,
-    merge,
-    mergeBlockConcurrency,
-    mergeLimiter = defaultMergeLimiter,
-    logInfo = noop,
-    logWarn = console.warn,
-  }
-) {
-  const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
-
-  const handler = this._handler
-
-  const vhdsToJSons = new Set()
-  const vhdById = new Map()
-  const vhdParents = { __proto__: null }
-  const vhdChildren = { __proto__: null }
-
-  const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
-
-  // from 5.110 to 5.113 we computed checksum for vhd file
-  // but never used nor removed them
-  await asyncMap(checksums, async path => {
-    if (remove) {
-      logInfo('deleting checksum file ', { path })
-      return handler.unlink(path)
-    }
-  })
-
-  // remove broken VHDs
-  await asyncMap(vhds, async path => {
-    if (removeTmp && basename(path)[0] === '.') {
-      logInfo('deleting temporary VHD', { path })
-      return VhdAbstract.unlink(handler, path)
-    }
-    try {
-      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
-        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
-          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-          vhdParents[path] = parent
-          if (parent in vhdChildren) {
-            const error = new Error('this script does not support multiple VHD children')
-            error.parent = parent
-            error.child1 = vhdChildren[parent]
-            error.child2 = path
-            throw error // should we throw?
-          }
-          vhdChildren[parent] = path
-        }
-        // Detect VHDs with the same UUIDs
-        //
-        // Due to a bug introduced in a1bcd35e2
-        const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
-        let vhdKept = vhd
-        if (duplicate !== undefined) {
-          logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
-          if (duplicate.containsAllDataOf(vhd)) {
-            logWarn(`should delete ${path}`)
-            vhdKept = duplicate
-            vhds.delete(path)
-          } else if (vhd.containsAllDataOf(duplicate)) {
-            logWarn(`should delete ${duplicate._path}`)
-            vhds.delete(duplicate._path)
-          } else {
-            logWarn('same ids but different content')
-          }
-        }
-        vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
-      })
-    } catch (error) {
-      vhds.delete(path)
-      logWarn('VHD check error', { path, error })
-      if (error?.code === 'ERR_ASSERTION' && remove) {
-        logInfo('deleting broken VHD', { path })
-        return VhdAbstract.unlink(handler, path)
-      }
-    }
-  })
-
-  // remove interrupted merge states for missing VHDs
-  for (const interruptedVhd of interruptedVhds.keys()) {
-    if (!vhds.has(interruptedVhd)) {
-      const { statePath } = interruptedVhds.get(interruptedVhd)
-      interruptedVhds.delete(interruptedVhd)
-
-      logWarn('orphan merge state', {
-        mergeStatePath: statePath,
-        missingVhdPath: interruptedVhd,
-      })
-      if (remove) {
-        logInfo('deleting orphan merge state', { statePath })
-        await handler.unlink(statePath)
-      }
-    }
-  }
-
-  // check if alias are correct
-  // check if all vhd in data subfolder have a corresponding alias
-  await asyncMap(Object.keys(aliases), async dir => {
-    await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
-  })
-
-  // remove VHDs with missing ancestors
-  {
-    const deletions = []
-
-    // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhdPath => {
-      const parent = vhdParents[vhdPath]
-      if (parent === undefined) {
-        return
-      }
-
-      // no longer needs to be checked
-      delete vhdParents[vhdPath]
-
-      deleteIfOrphan(parent)
-
-      if (!vhds.has(parent)) {
-        vhds.delete(vhdPath)
-
-        logWarn('parent VHD is missing', { parent, child: vhdPath })
-        if (remove) {
-          logInfo('deleting orphan VHD', { path: vhdPath })
-          deletions.push(VhdAbstract.unlink(handler, vhdPath))
-        }
-      }
-    }
-
-    // > A property that is deleted before it has been visited will not be
-    // > visited later.
-    // >
-    // > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
-    for (const child in vhdParents) {
-      deleteIfOrphan(child)
-    }
-
-    await Promise.all(deletions)
-  }
-
-  const jsons = new Set()
-  const xvas = new Set()
-  const xvaSums = []
-  const entries = await handler.list(vmDir, {
-    prependDir: true,
-  })
-  entries.forEach(path => {
-    if (isMetadataFile(path)) {
-      jsons.add(path)
-    } else if (isXvaFile(path)) {
-      xvas.add(path)
-    } else if (isXvaSumFile(path)) {
-      xvaSums.push(path)
-    }
-  })
-
-  const cachePath = vmDir + '/cache.json.gz'
-
-  let mustRegenerateCache
-  {
-    const cache = await this._readCache(cachePath)
-    const actual = cache === undefined ? 0 : Object.keys(cache).length
-    const expected = jsons.size
-
-    mustRegenerateCache = actual !== expected
-    if (mustRegenerateCache) {
-      logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
-    }
-  }
-
-  await asyncMap(xvas, async path => {
-    // check is not good enough to delete the file, the best we can do is report
-    // it
-    if (!(await this.isValidXva(path))) {
-      logWarn('XVA might be broken', { path })
-    }
-  })
-
-  const unusedVhds = new Set(vhds)
-  const unusedXvas = new Set(xvas)
-
-  const backups = new Map()
-
-  // compile the list of unused XVAs and VHDs, and remove backup metadata which
-  // reference a missing XVA/VHD
-  await asyncMap(jsons, async json => {
-    let metadata
-    try {
-      metadata = JSON.parse(await handler.readFile(json))
-    } catch (error) {
-      logWarn('failed to read backup metadata', { path: json, error })
-      jsons.delete(json)
-      return
-    }
-
-    let isBackupComplete
-
-    const { mode } = metadata
-    if (mode === 'full') {
-      const linkedXva = resolve('/', vmDir, metadata.xva)
-      isBackupComplete = xvas.has(linkedXva)
-      if (isBackupComplete) {
-        unusedXvas.delete(linkedXva)
-      } else {
-        logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
-      }
-    } else if (mode === 'delta') {
-      const linkedVhds = (() => {
-        const { vhds } = metadata
-        return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
-      })()
-
-      const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
-      isBackupComplete = missingVhds.length === 0
-
-      // FIXME: find better approach by keeping as much of the backup as
-      // possible (existing disks) even if one disk is missing
-      if (isBackupComplete) {
-        linkedVhds.forEach(_ => unusedVhds.delete(_))
-        linkedVhds.forEach(path => {
-          vhdsToJSons[path] = json
-        })
-      } else {
-        logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
-      }
-    }
-
-    if (isBackupComplete) {
-      backups.set(json, metadata)
-    } else {
-      jsons.delete(json)
-      if (remove) {
-        logInfo('deleting incomplete backup', { backup: json })
-        mustRegenerateCache = true
-        await handler.unlink(json)
-      }
-    }
-  })
-
-  // TODO: parallelize by vm/job/vdi
-  const unusedVhdsDeletion = []
-  const toMerge = []
-  {
-    // VHD chains (as list from oldest to most recent) to merge indexed by most recent
-    // ancestor
-    const vhdChainsToMerge = { __proto__: null }
-
-    const toCheck = new Set(unusedVhds)
-
-    const getUsedChildChainOrDelete = vhd => {
-      if (vhd in vhdChainsToMerge) {
-        const chain = vhdChainsToMerge[vhd]
-        delete vhdChainsToMerge[vhd]
-        return chain
-      }
-
-      if (!unusedVhds.has(vhd)) {
-        return [vhd]
-      }
-
-      // no longer needs to be checked
-      toCheck.delete(vhd)
-
-      const child = vhdChildren[vhd]
-      if (child !== undefined) {
-        const chain = getUsedChildChainOrDelete(child)
-        if (chain !== undefined) {
-          chain.unshift(vhd)
-          return chain
-        }
-      }
-
-      // no warning because a VHD can be unused for perfectly good reasons,
-      // e.g. the corresponding backup (metadata file) has been deleted
-      if (remove) {
-        logInfo('deleting unused VHD', { path: vhd })
-        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
-      }
-    }
-
-    toCheck.forEach(vhd => {
-      vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
-    })
-
-    // merge interrupted VHDs
-    for (const parent of interruptedVhds.keys()) {
-      // before #6349 the chain wasn't in the mergeState
-      const { chain, statePath } = interruptedVhds.get(parent)
-      if (chain === undefined) {
-        vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
-      } else {
-        vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
-      }
-    }
-
-    Object.values(vhdChainsToMerge).forEach(chain => {
-      if (chain !== undefined) {
-        toMerge.push(chain)
-      }
-    })
-  }
-
-  let totalMergedDataSize = 0
-  const metadataWithMergedVhd = {}
-  const doMerge = async () => {
-    await asyncMap(toMerge, async chain => {
-      const { finalDiskSize, mergedDataSize } = await limitedMergeVhdChain(handler, chain, {
-        logInfo,
-        logWarn,
-        remove,
-        mergeBlockConcurrency,
-      })
-      totalMergedDataSize += mergedDataSize
-      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
-      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
-    })
-
-    return { size: totalMergedDataSize }
-  }
-
-  await Promise.all([
-    ...unusedVhdsDeletion,
-    toMerge.length !== 0 && (merge ? Task.run({ properties: { name: 'merge' } }, doMerge) : () => Promise.resolve()),
-    asyncMap(unusedXvas, path => {
-      logWarn('unused XVA', { path })
-      if (remove) {
-        logInfo('deleting unused XVA', { path })
-        return handler.unlink(path)
-      }
-    }),
-    asyncMap(xvaSums, path => {
-      // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
-      if (!xvas.has(path.slice(0, -'.checksum'.length))) {
-        logInfo('unused XVA checksum', { path })
-        if (remove) {
-          logInfo('deleting unused XVA checksum', { path })
-          return handler.unlink(path)
-        }
-      }
-    }),
-  ])
-
-  // update size for delta metadata with merged VHD
-  // check for the other that the size is the same as the real file size
-  await asyncMap(jsons, async metadataPath => {
-    const metadata = backups.get(metadataPath)
-
-    let fileSystemSize
-    const mergedSize = metadataWithMergedVhd[metadataPath]
-
-    const { mode, size, xva } = metadata
-
-    try {
-      if (mode === 'full') {
-        // a full backup : check size
-        const linkedXva = resolve('/', vmDir, xva)
-        try {
-          fileSystemSize = await handler.getSize(linkedXva)
-          if (fileSystemSize !== size && fileSystemSize !== undefined) {
-            logWarn('cleanVm: incorrect backup size in metadata', {
-              path: metadataPath,
-              actual: size ?? 'none',
-              expected: fileSystemSize,
-            })
-          }
-        } catch (error) {
-          // will fail with encrypted remote
-        }
-      }
-    } catch (error) {
-      logWarn('failed to get backup size', { backup: metadataPath, error })
-      return
-    }
-
-    // systematically update size and differentials after a merge
-
-    // @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
-    if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
-      metadata.size = mergedSize ?? fileSystemSize ?? size
-
-      if (mergedSize) {
-        // all disks are now key disk
-        metadata.isVhdDifferencing = {}
-        for (const id of Object.keys(metadata.vdis ?? {})) {
-          metadata.isVhdDifferencing[id] = false
-        }
-      }
-      mustRegenerateCache = true
-      try {
-        await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
-      } catch (error) {
-        logWarn('failed to update backup size in metadata', { path: metadataPath, error })
-      }
-    }
-  })
-
-  if (mustRegenerateCache) {
-    const cache = {}
-    for (const [path, content] of backups.entries()) {
-      cache[path] = {
-        _filename: path,
-        id: path,
-        ...content,
-      }
-    }
-    await this._writeCache(cachePath, cache)
-  }
-
-  return {
-    // boolean whether some VHDs were merged (or should be merged)
-    merge: toMerge.length !== 0,
-    size: totalMergedDataSize,
-  }
-}

@xen-orchestra/backups/_fileRestore.integ.mjs

@@ -0,0 +1,126 @@
+import { strict as assert } from 'node:assert'
+import test from 'node:test'
+import { execFile } from 'node:child_process'
+import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { Stream } from 'node:stream'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { randomBytes } from 'node:crypto'
+
+import Disposable from 'promise-toolbox/Disposable'
+
+import { RemoteAdapter } from './RemoteAdapter.mjs'
+
+// losetup/dmsetup/pvcreate/vgcreate/lvcreate/mkfs all require root (CAP_SYS_ADMIN).
+const skip =
+  typeof process.getuid !== 'function' || process.getuid() !== 0 ? 'requires root (losetup/dmsetup/lvm)' : false
+
+const pExec = promisify(execFile)
+const uniqueName = () => `xotest${randomBytes(6).toString('hex')}`
+
+async function withLoop(imagePath, fn) {
+  const loop = (await pExec('losetup', ['--show', '-f', imagePath])).stdout.trim()
+  try {
+    return await fn(loop)
+  } finally {
+    await pExec('losetup', ['-d', loop]).catch(() => {})
+  }
+}
+
+// Build a self-contained LVM image: a 64 MiB file holding one PV / VG / ext4 LV.
+async function createLvmImage(imagePath, vgName, lvName) {
+  await pExec('truncate', ['-s', '64M', imagePath])
+  await withLoop(imagePath, async loop => {
+    await pExec('pvcreate', ['-f', '-y', loop])
+    await pExec('vgcreate', [vgName, loop])
+    await pExec('lvcreate', ['-y', '-l', '100%FREE', '-n', lvName, vgName])
+    await pExec('mkfs.ext4', ['-q', `/dev/${vgName}/${lvName}`])
+    await pExec('vgchange', ['-an', vgName])
+  })
+}
+
+// xo-proxy answers this stream through Koa, which pipes only bodies passing `instanceof Stream`
+// and JSON-serializes anything else: node-tar >= 7 returns a duck-typed (Minipass) stream, which
+// silently turned a proxied file restore into a JSON dump of the tar object. Every format must
+// resolve with a real node stream.
+test('fetchPartitionFiles resolves with a node:stream for every format', async t => {
+  const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-fetch-'))
+  t.after(() => rm(tmp, { recursive: true, force: true }))
+  await writeFile(join(tmp, 'file.txt'), 'hello')
+
+  const adapter = new RemoteAdapter({})
+  // at this point of a real restore the partition is already mounted
+  adapter.getPartition = Disposable.factory(async function* () {
+    yield tmp
+  })
+
+  for (const [format, magic] of [
+    ['tgz', Buffer.from([0x1f, 0x8b])],
+    ['zip', Buffer.from('PK')],
+  ]) {
+    const stream = await adapter.fetchPartitionFiles('disk', 'partition', ['/file.txt'], format)
+    assert.ok(stream instanceof Stream, `${format}: expected a node:stream instance`)
+
+    const chunks = []
+    for await (const chunk of stream) {
+      chunks.push(chunk)
+    }
+    const archive = Buffer.concat(chunks)
+    assert.deepEqual(archive.subarray(0, magic.length), magic, `${format}: unexpected archive header`)
+  }
+})
+
+test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => {
+  const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-'))
+  const adapter = new RemoteAdapter({})
+  const vg = uniqueName()
+  const lv = 'root_lv'
+  const image = join(tmp, 'disk.raw')
+  const clone = join(tmp, 'clone.raw')
+  const plain = join(tmp, 'plain.raw')
+
+  t.after(async () => {
+    await rm(tmp, { recursive: true, force: true })
+    // drop any stale online-cache entries pointing at the now-deleted images
+    await pExec('pvscan', ['--cache']).catch(() => {})
+  })
+
+  await createLvmImage(image, vg, lv)
+
+  await t.test('detects the PV and renames the VG to a unique name', async () => {
+    await Disposable.use(adapter._getLvmPhysicalVolume(image, undefined), async ({ originalVgName, vgName }) => {
+      assert.equal(originalVgName, vg)
+      assert.match(vgName, /^xo[0-9a-f]{16}$/)
+      assert.notEqual(vgName, vg)
+    })
+  })
+
+  await t.test('two clones of the same VM (shared PVID) list concurrently without collision', async () => {
+    await copyFile(image, clone) // exact clone → identical PVID + VG name + VG UUID
+    const [a, b] = await Promise.all([
+      adapter._listLvmLogicalVolumes(image, undefined, []),
+      adapter._listLvmLogicalVolumes(clone, undefined, []),
+    ])
+    assert.equal(a.length, 1)
+    assert.equal(b.length, 1)
+    // the human-readable name is the (shared) original; the id carries the unique renamed VG
+    assert.equal(a[0].name, `${vg}/${lv}`)
+    assert.equal(b[0].name, `${vg}/${lv}`)
+    assert.notEqual(a[0].id, b[0].id)
+  })
+
+  await t.test('a non-LVM partition is rejected without building a snapshot', async () => {
+    await pExec('truncate', ['-s', '16M', plain])
+    await pExec('mkfs.ext4', ['-q', plain])
+    await assert.rejects(
+      Disposable.use(adapter._getLvmPhysicalVolume(plain, undefined), async () => {}),
+      /no LVM physical volume/
+    )
+  })
+
+  await t.test('leaves no dangling xo-pv-* dm-snapshots behind', async () => {
+    const { stdout } = await pExec('dmsetup', ['ls']).catch(() => ({ stdout: '' }))
+    assert.equal(/xo-pv-/.test(stdout), false, `leftover snapshots:\n${stdout}`)
+  })
+})

@xen-orchestra/backups/_fileRestore.mjs

@@ -0,0 +1,485 @@
+// File-level-restore (FLR) methods for RemoteAdapter.
+//
+// These were extracted from RemoteAdapter.mjs to keep that file focused on backup
+// CRUD/cache/metadata. They are mixed back onto `RemoteAdapter.prototype` via
+// `Object.assign` and decorated via `decorateMethodsWith` in RemoteAdapter.mjs, so `this`
+// is the RemoteAdapter instance at call time (`this._handler`, `this._debounceResource`,
+// `this._useGetDiskLegacy` keep working unchanged). The logger keeps the original
+// `xo:backups:RemoteAdapter` namespace so existing debug filters are unaffected.
+import { asyncEach } from '@vates/async-each'
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+import { compose } from '@vates/compose'
+import { createLogger } from '@xen-orchestra/log'
+import { deduped } from '@vates/disposable/deduped.js'
+import { randomBytes } from 'node:crypto'
+import { Readable } from 'node:stream'
+import { join, resolve } from 'node:path'
+import { execFile } from 'child_process'
+import { finished } from 'node:stream/promises'
+import { lstat, open, readdir, unlink } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { mount } from '@vates/fuse-vhd'
+import { ZipFile } from 'yazl'
+import Disposable from 'promise-toolbox/Disposable'
+import fromCallback from 'promise-toolbox/fromCallback'
+import pDefer from 'promise-toolbox/defer'
+import * as tar from 'tar'
+
+import { getTmpDir } from './_getTmpDir.mjs'
+import {
+  listPartitions,
+  LINUX_DATA_PARTITION_TYPE_GPT,
+  LINUX_DATA_PARTITION_TYPE_MBR,
+  LVM_PARTITION_TYPE_GPT,
+  LVM_PARTITION_TYPE_MBR,
+} from './_listPartitions.mjs'
+import { lvs, pvs } from './_lvm.mjs'
+
+const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
+
+const noop = Function.prototype
+
+// Restrict LVM scanning to a single device. Backup PVs are clones: the very same PVID
+// appears at once on the raw loop device, the dm-snapshot overlaid on it, and every
+// other restored copy of the same VM. An unscoped pvs/pvscan/vgimportclone/vgchange then
+// aborts with "duplicate PV ... for PVID ..." and the VG can neither be renamed nor
+// activated. Accepting only the device in hand removes every duplicate from LVM's view.
+export const lvmOnlyDevice = devicePath => `devices { global_filter=[ "a|^${devicePath}$|", "r|.*|" ] }`
+
+// Partition-type predicates (exported for unit tests).
+export const isLvmPartitionType = type => type === LVM_PARTITION_TYPE_MBR || type === LVM_PARTITION_TYPE_GPT
+
+// Some installers (e.g. Ubuntu subiquity) place an LVM PV on a generic Linux-data partition
+// instead of the LVM type, so those are the only non-LVM types worth the (expensive) probe.
+export const isProbeableForLvm = type =>
+  type === LINUX_DATA_PARTITION_TYPE_MBR || type === LINUX_DATA_PARTITION_TYPE_GPT
+
+// Build the partition entries for a PV's logical volumes (exported for unit tests).
+// Shows "ubuntu-vg/ubuntu-lv" for readability; skips unnamed LVs.
+export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
+  lvItems
+    .filter(lv => lv.lv_name !== '')
+    .map(lv => ({
+      id: `${partitionId}/${lv.vg_name}/${lv.lv_name}`,
+      name: originalVgName !== undefined ? `${originalVgName}/${lv.lv_name}` : lv.lv_name,
+      size: lv.lv_size,
+    }))
+
+const makeRelative = path => resolve('/', path).slice(1)
+const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
+
+async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
+  for (const relativePath of relativePaths) {
+    const realPath = join(realBasePath, relativePath)
+    const virtualPath = join(virtualBasePath, relativePath)
+
+    const stats = await lstat(realPath)
+    const { mode, mtime } = stats
+    const opts = { mode, mtime }
+    if (stats.isDirectory()) {
+      zip.addEmptyDirectory(virtualPath, opts)
+      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
+    } else if (stats.isFile()) {
+      zip.addFile(realPath, virtualPath, opts)
+    }
+  }
+}
+
+const debounceResourceFactory = factory =>
+  function () {
+    return this._debounceResource(factory.apply(this, arguments))
+  }
+
+// legacy disk mount via vhdimount; extracted from a private method so it can live in this
+// module (mixin methods cannot reference class-private members).
+async function* getDiskLegacy(handler, diskId) {
+  const RE_VHDI = /^vhdi(\d+)$/
+
+  const diskPath = handler.getFilePath('/' + diskId)
+  const mountDir = yield getTmpDir()
+
+  debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
+  await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
+  try {
+    let max = 0
+    let maxEntry
+    const entries = await readdir(mountDir)
+    entries.forEach(entry => {
+      const matches = RE_VHDI.exec(entry)
+      if (matches !== null) {
+        const value = +matches[1]
+        if (value > max) {
+          max = value
+          maxEntry = entry
+        }
+      }
+    })
+    if (max === 0) {
+      throw new Error('no disks found')
+    }
+
+    yield `${mountDir}/${maxEntry}`
+  } finally {
+    debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
+    await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
+  }
+}
+
+// Mixed onto RemoteAdapter.prototype — `this` is the RemoteAdapter instance.
+export const fileRestoreMethods = {
+  async _findPartition(devicePath, partitionId) {
+    const partitions = await listPartitions(devicePath)
+    const partition = partitions.find(_ => _.id === partitionId)
+    if (partition === undefined) {
+      throw new Error(`partition ${partitionId} not found`)
+    }
+    return partition
+  },
+
+  async *_getLvmLogicalVolumes(devicePath, pvId, requestedVgName) {
+    const { vgName, lvmConfig } = yield this._getLvmPhysicalVolume(
+      devicePath,
+      pvId && (await this._findPartition(devicePath, pvId))
+    )
+
+    // vgName is the unique name vgimportclone just assigned to this PV; prefer it over the
+    // (possibly stale) name embedded in the partitionId. lvmConfig scopes every command to
+    // this device so a colliding/duplicate VG on the host or another copy can't shadow it.
+    const effectiveVgName = vgName ?? requestedVgName
+
+    debug('activate LVM volume group', { effectiveVgName, requestedVgName })
+    await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-ay', effectiveVgName])
+    try {
+      debug('get LVM logical volumes', { effectiveVgName })
+      yield lvs(['lv_name', 'lv_path'], '--config', lvmConfig, effectiveVgName)
+    } finally {
+      debug('deactivate LVM volume group', { effectiveVgName })
+      await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', effectiveVgName])
+    }
+  },
+
+  async *_getLvmPhysicalVolume(devicePath, partition) {
+    const loopArgs = []
+    if (partition !== undefined) {
+      loopArgs.push('-o', partition.start * 512, '--sizelimit', partition.size)
+    }
+    loopArgs.push('--show', '-f', devicePath)
+
+    debug('attach loop device', { devicePath, partition })
+    const loopDevice = (await fromCallback(execFile, 'losetup', loopArgs)).trim()
+
+    let cowPath, cowLoop, mapperName, vgName, lvmConfig
+    try {
+      // Cheap pre-check, scoped to this loop so a duplicate PVID (another copy of the same
+      // VM restored concurrently) can't make it fail: is there an LVM PV here at all?
+      // Non-PV partitions (/boot, EFI, raw disks) skip the whole dm-snapshot machinery.
+      const [originalVgName] = (
+        await pvs('vg_name', '--config', lvmOnlyDevice(loopDevice), loopDevice).catch(() => [])
+      ).filter(Boolean)
+      if (originalVgName === undefined) {
+        const where = partition !== undefined ? ` partition ${partition.id}` : ''
+        throw new Error(`no LVM physical volume on ${devicePath}${where}`)
+      }
+
+      // The backup is read-only, so overlay a writable dm-snapshot to let vgimportclone
+      // rewrite metadata, and rename the VG to a unique name so concurrent clones (identical
+      // PVID and VG name) don't collide on device-mapper node names. Every LVM command is
+      // scoped to the snapshot device (lvmConfig) to keep duplicate PVIDs out of LVM's view.
+      mapperName = `xo-pv-${randomBytes(4).toString('hex')}`
+      const mapperPath = `/dev/mapper/${mapperName}`
+      lvmConfig = lvmOnlyDevice(mapperPath)
+
+      // ~4 MB sparse COW is enough to hold the LVM metadata rewrites
+      cowPath = join(tmpdir(), `${mapperName}.cow`)
+      const fh = await open(cowPath, 'w')
+      await fh.truncate(4 * 1024 * 1024)
+      await fh.close()
+      cowLoop = (await fromCallback(execFile, 'losetup', ['--show', '-f', cowPath])).trim()
+
+      const sectors = (await fromCallback(execFile, 'blockdev', ['--getsz', loopDevice])).trim()
+      await fromCallback(execFile, 'dmsetup', [
+        'create',
+        mapperName,
+        '--table',
+        `0 ${sectors} snapshot ${loopDevice} ${cowLoop} P 8`,
+      ])
+
+      // Unique random VG name: list and mount each query/yield the name from the device, so
+      // it need not be deterministic — only collision-free across concurrent clones.
+      vgName = `xo${randomBytes(8).toString('hex')}`
+      debug('import LVM volume group with unique name via dm-snapshot', { vgName, originalVgName })
+      await fromCallback(execFile, 'vgimportclone', ['--config', lvmConfig, '--basevgname', vgName, mapperPath])
+
+      yield { path: mapperPath, originalVgName, vgName, lvmConfig }
+    } finally {
+      if (mapperName !== undefined) {
+        // best-effort deactivate (a no-op if it was never activated) before removing the snapshot
+        await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', vgName]).catch(noop)
+        debug('remove dm-snapshot', { mapperName })
+        await fromCallback(execFile, 'dmsetup', ['remove', mapperName]).catch(err =>
+          warn('failed to remove dm-snapshot', { mapperName, error: err })
+        )
+      }
+      if (cowLoop !== undefined) {
+        await fromCallback(execFile, 'losetup', ['-d', cowLoop]).catch(noop)
+      }
+      if (cowPath !== undefined) {
+        await unlink(cowPath).catch(noop)
+      }
+      debug('detach loop device', { loopDevice })
+      await fromCallback(execFile, 'losetup', ['-d', loopDevice])
+    }
+  },
+
+  async *_getPartition(devicePath, partition) {
+    const options = ['loop', 'ro']
+
+    if (partition !== undefined) {
+      const { size, start } = partition
+      options.push(`sizelimit=${size}`)
+      if (start !== undefined) {
+        options.push(`offset=${start * 512}`)
+      }
+    }
+
+    const path = yield getTmpDir()
+    const mount = options => {
+      debug('mount device', { devicePath, mountPath: path })
+      return fromCallback(execFile, 'mount', [
+        `--options=${options.join(',')}`,
+        `--source=${devicePath}`,
+        `--target=${path}`,
+      ]).catch(error => {
+        if (error.stderr) {
+          error.message = `${error.message}: ${error.stderr.trim()}`
+        }
+        throw error
+      })
+    }
+
+    // norecovery prevents mount from attempting journal replay on a read-only device (ext3/ext4/xfs).
+    // Other filesystems don't support it, so fall back without it on failure.
+    try {
+      await mount([...options, 'norecovery'])
+    } catch (error) {
+      await mount(options)
+    }
+
+    try {
+      yield path
+    } finally {
+      debug('umount device', { devicePath, mountPath: path })
+      await fromCallback(execFile, 'umount', ['--lazy', path])
+    }
+  },
+
+  _listLvmLogicalVolumes(devicePath, partition, results = []) {
+    return Disposable.use(
+      this._getLvmPhysicalVolume(devicePath, partition),
+      async ({ path, originalVgName, lvmConfig }) => {
+        const lvItems = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], '--config', lvmConfig, path)
+        const partitionId = partition !== undefined ? partition.id : ''
+        results.push(...toLvPartitions(partitionId, originalVgName, lvItems))
+        return results
+      }
+    )
+  },
+
+  fetchPartitionFiles(diskId, partitionId, paths, format) {
+    const { promise, reject, resolve } = pDefer()
+    const self = this
+    Disposable.use(async function* () {
+      const path = yield self.getPartition(diskId, partitionId)
+      let outputStream
+
+      if (format === 'tgz') {
+        // process one entry at a time with { job: 1}. node-tar defaults to 4
+        // concurrent jobs, which on a FUSE-backed restore mount means up to 4
+        // simultaneous reads. Those saturate the libuv threadpool and starve
+        // the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
+        // threadpool deadlock). Serializing keeps a worker free for them.
+        const pack = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
+
+        // node-tar >= 7 returns a Minipass stream: it has `pipe()` but is not a `node:stream`
+        // instance. xo-proxy serves this stream through Koa, which pipes only bodies passing
+        // `instanceof Stream` and JSON-serializes anything else — so a file restore through a
+        // proxy answered a JSON dump of the tar object instead of the archive. Wrapping it in a
+        // real node stream also propagates tar errors to the consumer and aborts the tar job
+        // when the client disconnects.
+        outputStream = Readable.from(pack, { objectMode: false })
+        resolve(outputStream)
+      } else if (format === 'zip') {
+        const zip = new ZipFile()
+        // Resolve with the stream before enumeration so the client can start
+        // receiving data immediately — addZipEntries over FUSE/S3 can take
+        // minutes for large trees (e.g. node_modules) and would otherwise
+        // appear as a freeze with no response sent.
+        outputStream = zip.outputStream
+        resolve(zip.outputStream)
+        await addZipEntries(zip, path, '', paths.map(makeRelative))
+        zip.end()
+      } else {
+        throw new Error('unsupported format ' + format)
+      }
+
+      await finished(outputStream).catch(noop)
+    }).catch(error => {
+      warn(error)
+      reject(error)
+    })
+    return promise
+  },
+
+  async *getDisk(diskId) {
+    if (this._useGetDiskLegacy) {
+      yield* getDiskLegacy(this._handler, diskId)
+      return
+    }
+    const handler = this._handler
+    // this is a disposable
+    const mountDir = yield getTmpDir()
+    // this is also a disposable
+    yield mount(handler, diskId, mountDir)
+    // this will yield disk path to caller
+    yield `${mountDir}/vhd0`
+  },
+
+  // partitionId values:
+  //
+  // - undefined: raw disk
+  // - `<partitionId>`: partitioned disk
+  // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
+  // - `/<vgName>/lvName>`: LVM on a raw disk
+  async *getPartition(diskId, partitionId) {
+    const devicePath = yield this.getDisk(diskId)
+    if (partitionId === undefined) {
+      debug(
+        'no partition specified, attempting raw disk mount — call listPartitions first if disk has partitions or LVM'
+      )
+      return yield this._getPartition(devicePath)
+    }
+
+    const isLvmPartition = partitionId.includes('/')
+    if (isLvmPartition) {
+      const [pvId, vgName, lvName] = partitionId.split('/')
+      const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
+      return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
+    }
+
+    return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
+  },
+
+  listPartitionFiles(diskId, partitionId, path) {
+    return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
+      path = resolveSubpath(rootPath, path)
+      const entriesMap = {}
+      await asyncEach(
+        await readdir(path),
+        async name => {
+          try {
+            const stats = await lstat(`${path}/${name}`)
+            if (stats.isDirectory()) {
+              entriesMap[name + '/'] = {}
+            } else if (stats.isFile()) {
+              entriesMap[name] = {}
+            }
+          } catch (error) {
+            if (error == null || error.code !== 'ENOENT') {
+              throw error
+            }
+          }
+        },
+        { concurrency: 1 }
+      )
+
+      return entriesMap
+    })
+  },
+
+  listPartitions(diskId) {
+    return Disposable.use(this.getDisk(diskId), async devicePath => {
+      // partx may return empty on FUSE-backed files (vhd0); a loop device
+      // presents proper block-device semantics that partx reads reliably.
+      // losetup may itself fail if the FUSE mount isn't fully ready yet —
+      // fall back to the direct path so listPartitions returns [] instead of throwing.
+      let loopForParts, partitions
+      try {
+        loopForParts = (await fromCallback(execFile, 'losetup', ['--show', '-f', devicePath])).trim()
+        partitions = await listPartitions(loopForParts)
+      } catch (error) {
+        debug('partition probe via loop device failed, falling back to direct path', { error })
+        partitions = await listPartitions(devicePath)
+      } finally {
+        if (loopForParts !== undefined) {
+          await fromCallback(execFile, 'losetup', ['-d', loopForParts]).catch(noop)
+        }
+      }
+
+      if (partitions.length === 0) {
+        try {
+          // handle potential raw LVM physical volume
+          return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
+        } catch (error) {
+          return []
+        }
+      }
+
+      const results = []
+      await asyncMapSettled(partitions, async partition => {
+        if (isLvmPartitionType(partition.type)) {
+          return this._listLvmLogicalVolumes(devicePath, partition, results)
+        }
+
+        // Only generic Linux-data partitions can hide an LVM PV (subiquity-style); other
+        // types — BIOS boot, EFI, swap, … — are never PVs, so list them directly without
+        // the (expensive) loop + dm-snapshot + vgimportclone probe.
+        if (!isProbeableForLvm(partition.type)) {
+          results.push(partition)
+          return
+        }
+
+        const lvResults = []
+        try {
+          await this._listLvmLogicalVolumes(devicePath, partition, lvResults)
+        } catch (error) {
+          debug('LVM probe failed for Linux-data partition, treating as regular partition', {
+            partition,
+            error,
+          })
+        }
+        if (lvResults.length > 0) {
+          results.push(...lvResults)
+        } else {
+          results.push(partition)
+        }
+      })
+      return results
+    })
+  },
+}
+
+// Applied by RemoteAdapter.mjs via decorateMethodsWith(RemoteAdapter, fileRestoreDecorators).
+// debounceResourceFactory/deduped reference `this._debounceResource` at call time.
+export const fileRestoreDecorators = {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+}

@xen-orchestra/backups/_fileRestore.test.mjs

@@ -0,0 +1,61 @@
+import { strict as assert } from 'node:assert'
+import test from 'node:test'
+
+import { isLvmPartitionType, isProbeableForLvm, lvmOnlyDevice, toLvPartitions } from './_fileRestore.mjs'
+import {
+  LINUX_DATA_PARTITION_TYPE_GPT,
+  LINUX_DATA_PARTITION_TYPE_MBR,
+  LVM_PARTITION_TYPE_GPT,
+  LVM_PARTITION_TYPE_MBR,
+} from './_listPartitions.mjs'
+
+// well-known GPT type GUIDs that must NOT be probed/treated as LVM
+const EFI = 'c12a7328-f81f-11d2-ba4b-00a0c93ec93b'
+const BIOS_BOOT = '21686148-6449-6e6f-744e-656564454649'
+const SWAP = '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f'
+
+test('lvmOnlyDevice builds an accept-only global_filter for the device', () => {
+  assert.equal(
+    lvmOnlyDevice('/dev/mapper/xo-pv-abcd1234'),
+    'devices { global_filter=[ "a|^/dev/mapper/xo-pv-abcd1234$|", "r|.*|" ] }'
+  )
+  assert.equal(lvmOnlyDevice('/dev/loop7'), 'devices { global_filter=[ "a|^/dev/loop7$|", "r|.*|" ] }')
+})
+
+test('isLvmPartitionType matches only the LVM partition types', () => {
+  assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_MBR), true)
+  assert.equal(isLvmPartitionType(LVM_PARTITION_TYPE_GPT), true)
+  assert.equal(isLvmPartitionType(LINUX_DATA_PARTITION_TYPE_GPT), false)
+  assert.equal(isLvmPartitionType(EFI), false)
+  assert.equal(isLvmPartitionType(undefined), false)
+})
+
+test('isProbeableForLvm matches only generic Linux-data types', () => {
+  assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_MBR), true)
+  assert.equal(isProbeableForLvm(LINUX_DATA_PARTITION_TYPE_GPT), true)
+  // never probe these
+  assert.equal(isProbeableForLvm(LVM_PARTITION_TYPE_GPT), false)
+  assert.equal(isProbeableForLvm(BIOS_BOOT), false)
+  assert.equal(isProbeableForLvm(EFI), false)
+  assert.equal(isProbeableForLvm(SWAP), false)
+  assert.equal(isProbeableForLvm(undefined), false)
+})
+
+test('toLvPartitions builds entries, skips unnamed LVs, shows the original VG name', () => {
+  const lvItems = [
+    { lv_name: 'ubuntu-lv', lv_path: '/dev/xohash/ubuntu-lv', lv_size: '1000', vg_name: 'xohash' },
+    // a PV with no LV yields a row with an empty lv_name → must be skipped
+    { lv_name: '', lv_path: '', lv_size: '0', vg_name: 'xohash' },
+    { lv_name: 'swap_1', lv_path: '/dev/xohash/swap_1', lv_size: '500', vg_name: 'xohash' },
+  ]
+  assert.deepEqual(toLvPartitions('part3', 'ubuntu-vg', lvItems), [
+    { id: 'part3/xohash/ubuntu-lv', name: 'ubuntu-vg/ubuntu-lv', size: '1000' },
+    { id: 'part3/xohash/swap_1', name: 'ubuntu-vg/swap_1', size: '500' },
+  ])
+})
+
+test('toLvPartitions falls back to the bare LV name when the original VG name is unknown', () => {
+  const lvItems = [{ lv_name: 'data', lv_path: '/dev/xohash/data', lv_size: '42', vg_name: 'xohash' }]
+  // raw-disk PV → empty partitionId
+  assert.deepEqual(toLvPartitions('', undefined, lvItems), [{ id: '/xohash/data', name: 'data', size: '42' }])
+})

@xen-orchestra/backups/_incrementalVm.mjs

@@ -16,6 +16,44 @@ import { toQcow2Stream } from '@xen-orchestra/qcow2'
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
 
+const orderedMemoryLimits = ['memory_static_min', 'memory_dynamic_min', 'memory_dynamic_max', 'memory_static_max']
+
+// The dynamic memory range MUST respect this inequality at any moment: static_min <= dynamic_min <= dynamic_max <= static_max.
+// We must update these properties in the right order to avoid XAPI error.
+// The order depends on the values. It can be an increase, a decrease or a mix of both, so any order could be required.
+export async function updateMemoryFields(xapi, targetVm, vmRecord) {
+  const memoryValues = {}
+  for (const key of orderedMemoryLimits) {
+    memoryValues[key] = {
+      currentValue: targetVm[key],
+      newValue: vmRecord[key] ?? targetVm[key],
+    }
+  }
+
+  while (await updateNextMemoryField(xapi, memoryValues, targetVm.$ref)) {
+    /* execute until all memory fields are updated */
+  }
+}
+
+// Update one more memory field if needed, then return a boolean describing if a field was updated or if all fields are up to date
+async function updateNextMemoryField(xapi, memoryValues, vmRef) {
+  for (let i = 0; i < orderedMemoryLimits.length; i++) {
+    const currentField = memoryValues[orderedMemoryLimits[i]]
+    const nextField = i === orderedMemoryLimits.length - 1 ? undefined : memoryValues[orderedMemoryLimits[i + 1]]
+    if (
+      currentField.newValue !== currentField.currentValue &&
+      (nextField === undefined || currentField.newValue <= nextField.currentValue)
+    ) {
+      // no need to check that previousField.currentValue <= currentField.newValue, as we can deduce it
+      await xapi.setField('VM', vmRef, orderedMemoryLimits[i], currentField.newValue)
+      await xapi.barrier()
+      currentField.currentValue = currentField.newValue
+      return true
+    }
+  }
+  return false
+}
+
 export async function exportIncrementalVm(
   vm,
   baseVdis = {},
@@ -160,6 +198,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   let vmRef
   if (isUpdate) {
     vmRef = targetRef
+
+    await updateMemoryFields(xapi, targetVm, vmRecord)
+
     await Promise.all([
       xapi.setFields('VM', vmRef, {
         actions_after_crash: vmRecord.actions_after_crash,
@@ -171,10 +212,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
         HVM_boot_params: vmRecord.HVM_boot_params,
         HVM_boot_policy: vmRecord.HVM_boot_policy,
         HVM_shadow_multiplier: vmRecord.HVM_shadow_multiplier,
-        memory_dynamic_max: vmRecord.memory_dynamic_max,
-        memory_dynamic_min: vmRecord.memory_dynamic_min,
-        memory_static_max: vmRecord.memory_static_max,
-        memory_static_min: vmRecord.memory_static_min,
         name_label: vmRecord.name_label,
         name_description: vmRecord.name_description,
         order: vmRecord.order,

@xen-orchestra/backups/_incrementalVm.test.mjs

@@ -0,0 +1,122 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { updateMemoryFields } from './_incrementalVm.mjs'
+
+const MOCK_REF = 'mockRef'
+
+function mockXapiVm(initialMemoryValues) {
+  const res = {
+    ...initialMemoryValues,
+    updates: [],
+    barrier: () => {},
+    setField: (type, ref, field, value) => {
+      assert(type === 'VM')
+      assert(ref === MOCK_REF)
+      res.updates.push({ field, value })
+      res[field] = value
+      assert(res.memory_static_min <= res.memory_dynamic_min)
+      assert(res.memory_dynamic_min <= res.memory_dynamic_max)
+      assert(res.memory_dynamic_max <= res.memory_static_max)
+    },
+  }
+  return res
+}
+
+const tests = [
+  {
+    label: 'handles a memory increase',
+    targetVm: {
+      memory_static_min: 2,
+      memory_dynamic_min: 4,
+      memory_dynamic_max: 4,
+      memory_static_max: 8,
+    },
+    vmRecord: {
+      memory_static_min: 16,
+      memory_dynamic_min: 16,
+      memory_dynamic_max: 16,
+      memory_static_max: 16,
+    },
+    expectedUpdates: [
+      { field: 'memory_static_max', value: 16 },
+      { field: 'memory_dynamic_max', value: 16 },
+      { field: 'memory_dynamic_min', value: 16 },
+      { field: 'memory_static_min', value: 16 },
+    ],
+  },
+  {
+    label: 'handles a memory decrease',
+    targetVm: {
+      memory_static_min: 4,
+      memory_dynamic_min: 8,
+      memory_dynamic_max: 16,
+      memory_static_max: 16,
+    },
+    vmRecord: {
+      memory_static_min: 2,
+      memory_dynamic_min: 2,
+      memory_dynamic_max: 2,
+      memory_static_max: 2,
+    },
+    expectedUpdates: [
+      { field: 'memory_static_min', value: 2 },
+      { field: 'memory_dynamic_min', value: 2 },
+      { field: 'memory_dynamic_max', value: 2 },
+      { field: 'memory_static_max', value: 2 },
+    ],
+  },
+  {
+    label: 'handles a more complex case',
+    targetVm: {
+      memory_static_min: 1,
+      memory_dynamic_min: 2,
+      memory_dynamic_max: 8,
+      memory_static_max: 16,
+    },
+    vmRecord: {
+      memory_static_min: 4,
+      memory_dynamic_min: 4,
+      memory_dynamic_max: 4,
+      memory_static_max: 4,
+    },
+    expectedUpdates: [
+      { field: 'memory_dynamic_min', value: 4 },
+      { field: 'memory_static_min', value: 4 },
+      { field: 'memory_dynamic_max', value: 4 },
+      { field: 'memory_static_max', value: 4 },
+    ],
+  },
+  {
+    label: 'avoids updating identical value',
+    targetVm: {
+      memory_static_min: 1,
+      memory_dynamic_min: 2,
+      memory_dynamic_max: 4,
+      memory_static_max: 8,
+    },
+    vmRecord: {
+      memory_static_min: 1,
+      memory_dynamic_min: 2,
+      memory_dynamic_max: 4,
+      memory_static_max: 8,
+    },
+    expectedUpdates: [],
+  },
+]
+
+describe('updateMemoryFields() should update in the right order', () => {
+  for (const test of tests) {
+    it(test.label, async () => {
+      const mock = mockXapiVm(test.targetVm)
+      test.targetVm.$ref = MOCK_REF
+      await updateMemoryFields(mock, test.targetVm, test.vmRecord)
+
+      assert.equal(mock.updates.length, test.expectedUpdates.length)
+      for (let i = 0; i < mock.updates.length; i++) {
+        assert.equal(mock.updates[i].value, test.expectedUpdates[i].value)
+        assert.equal(mock.updates[i].field, test.expectedUpdates[i].field)
+      }
+    })
+  }
+})

@xen-orchestra/backups/_listPartitions.mjs

@@ -29,6 +29,15 @@ export const LVM_PARTITION_TYPE_MBR = 0x8e
 // GPT LVM type
 export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
 
+// Generic "Linux filesystem data" types. These are the only non-LVM-typed
+// partitions worth probing for a hidden LVM PV, because some installers (e.g.
+// Ubuntu subiquity) place a PV on a partition left with this generic type.
+// Other types (BIOS boot, EFI, swap, …) are never LVM PVs.
+// MBR Linux native
+export const LINUX_DATA_PARTITION_TYPE_MBR = 0x83
+// GPT Linux filesystem data
+export const LINUX_DATA_PARTITION_TYPE_GPT = '0fc63daf-8483-4772-8e79-3d69d8477de4'
+
 const parsePartxLine = createParser({
   keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
   valueTransform: (value, key) => {

@xen-orchestra/backups/_otherConfig.mjs

@@ -33,6 +33,13 @@ export const VM_UUID = 'xo:backup:vm'
 // allows direction-agnostic discovery of common base VDIs (e.g. for failback/reverse replication)
 export const CONTENT_KEY = 'xo:backup:contentKey'
 
+// in `other_config` of a replicated VM and its VDIs
+// set to the string 'true' on every write done by code that includes the non-NBD qcow2
+// corruption fix (fix(disk-transform): erroneous DiskLargerBlock copy offset). Its absence
+// on a qcow2 full means the replica may be silently corrupt and its chain must be re-based
+// with a clean full. Named after the specific fix so future fixes get their own flag.
+export const INCLUDE_NON_NBD_QCOW2_FIX = 'xo:backup:includeNonNbdQcow2Fix'
+
 async function listVdiRefs(xapi, vmRef) {
   return xapi.VM_getDisks(vmRef)
 }
@@ -85,13 +92,16 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 export function resetVmOtherConfig(xapi, vmRef) {
   return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
     return xapi.setFieldEntries(type, ref, 'other_config', {
+      [CONTENT_KEY]: null,
       [COPY_OF]: null,
       [DATETIME]: null,
       [DELTA_CHAIN_LENGTH]: null,
       [EXPORTED_SUCCESSFULLY]: null,
+      [INCLUDE_NON_NBD_QCOW2_FIX]: null,
       [JOB_ID]: null,
       [SCHEDULE_ID]: null,
       [VM_UUID]: null,
+
       // REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
     })
   })
@@ -118,6 +128,10 @@ export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, schedule
     xapi.setFieldEntries(type, ref, 'other_config', {
       [REPLICATED_TO_SR_UUID]: srUuid,
       [DATETIME]: formatDateTime(timestamp),
+      // stamped on every write: its presence marks the object as produced by code
+      // that includes the non-NBD qcow2 corruption fix, so a chain rooted in a pre-fix
+      // qcow2 full (missing this flag) can be detected and re-based with a clean full
+      [INCLUDE_NON_NBD_QCOW2_FIX]: 'true',
       [JOB_ID]: jobId,
       [SCHEDULE_ID]: scheduleId,
       [VM_UUID]: vmUuid,