fix out-of-range block access in BlockAllocationTable.isEmpty

@xen-orchestra/backup-archive/.eslintrc.cjs

@@ -0,0 +1,13 @@
+'use strict'
+
+module.exports = {
+  overrides: [
+    {
+      // .mjs files importing compiled .mts sources (`./x.mjs` only exists in `dist/`)
+      files: ['src/**/*.mjs'],
+      rules: {
+        'n/no-missing-import': 'off',
+      },
+    },
+  ],
+}

@xen-orchestra/backup-archive/package.json

@@ -8,7 +8,8 @@
   "main": "./dist/index.mjs",
   "exports": {
     ".": "./dist/index.mjs",
-    "./disks": "./dist/disks/index.mjs"
+    "./disks": "./dist/disks/index.mjs",
+    "./disks/hashed": "./dist/disks/hashed/index.mjs"
   },
   "devDependencies": {
     "@eslint/js": "^9.19.0",
@@ -32,14 +33,16 @@
     "uuid": "^9.0.0",
     "promise-toolbox": "^0.21.0",
     "@vates/async-each": "^1.0.1",
-    "vhd-lib": "^4.16.1"
+    "vhd-lib": "^4.16.1",
+    "compare-versions": "^6.1.1",
+    "@vates/read-chunk": "^1.2.1"
   },
   "scripts": {
     "build": "yarn run clean && tsc",
     "dev": "tsc --watch",
     "postversion": "npm publish --access public",
-    "test": "yarn run clean && tsc && node --test **/*.test.mjs",
-    "test-integration": "tsc && node --test **/*.integ.mjs",
+    "test": "tsc && node --test \"**/*.test.mjs\"",
+    "test-integration": "tsc && node --test \"**/*.integ.mjs\"",
     "clean": "rimraf dist/"
   },
   "private": false,

@xen-orchestra/backup-archive/src/RemoteDiskLineage.mts

@@ -47,7 +47,7 @@ export class RemoteDiskLineage {
 
   constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
     this.#handler = handler
-    this.#vdiDir = vdiDir
+    this.#vdiDir = normalize(vdiDir)
     this.#opts = opts
   }
 

@xen-orchestra/backup-archive/src/VmBackupDirectory.mts

@@ -82,7 +82,7 @@ export class VmBackupDirectory implements VmBackupInterface {
     for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
       let metadata: PartialBackupMetadata | undefined
       try {
-        metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
+        metadata = JSON.parse((await this.handler.readFile(fullPath)).toString()) satisfies PartialBackupMetadata
       } catch (error) {
         this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
       }

@xen-orchestra/backup-archive/src/disks/MergeRemoteDisk.mjs

@@ -140,7 +140,7 @@ export class MergeRemoteDisk {
 
     try {
       const mergeStateContent = await this.#handler.readFile(this.#statePath)
-      this.#state = JSON.parse(mergeStateContent)
+      this.#state = JSON.parse(mergeStateContent.toString())
 
       // work-around to bugs introduced in 97d94b795 and 4e50858
       //
@@ -344,7 +344,7 @@ export class MergeRemoteDisk {
       const parentPath = normalize(vdiDir + '/' + match[1])
       let chain
       try {
-        const state = JSON.parse(await handler.readFile(filePath))
+        const state = JSON.parse((await handler.readFile(filePath)).toString())
         if (Array.isArray(state?.chain)) {
           chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
         } else {

@xen-orchestra/backup-archive/src/disks/RemoteDisk.mjs

@@ -2,14 +2,16 @@
 
 /**
  * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
+ * @typedef {import('@xen-orchestra/fs').DiskFormat} DiskFormat
  */
 
 import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
 export class RemoteDisk extends RandomAccessDisk {
   /**
    * Abstract
-   * @param {Object} options
-   * @param {boolean} options.force
+   * @param {Object} [options]
+   * @param {boolean} [options.force]
+   * @param {boolean} [options.ignoreBlockIndexes] skip reading the block index when the format allows it
    * @returns {Promise<void>}
    */
   async init(options = { force: false }) {
@@ -238,9 +240,10 @@ export class RemoteDisk extends RandomAccessDisk {
    * compression, ...) — calling `super.isMergeableParent()` first.
    *
    * @param {string} parentUuid
+   * @param {{ diskFormat: DiskFormat }} options format of the disks written to the remote
    * @returns {Promise<boolean>}
    */
-  async isMergeableParent(parentUuid) {
+  async isMergeableParent(parentUuid, { diskFormat }) {
     return this.getUuid() === parentUuid
   }
 

@xen-orchestra/backup-archive/src/disks/RemoteDiskChain.mjs

@@ -0,0 +1,275 @@
+// @ts-check
+
+/**
+ * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
+ */
+
+import { RemoteDisk } from './RemoteDisk.mjs'
+
+/**
+ * A chain of disks of the same format, oldest first, read as a single disk.
+ *
+ * Format agnostic: the format specific parts (size on disk, concurrent merge,
+ * block location) are implemented by the subclasses.
+ */
+export class RemoteDiskChain extends RemoteDisk {
+  /**
+   * @type {RemoteDisk[]}
+   */
+  #disks
+
+  /**
+   * @param {Object} params
+   * @param {RemoteDisk[]} params.disks
+   */
+  constructor({ disks }) {
+    super()
+    this.#disks = disks
+  }
+
+  /**
+   * The disks of the chain, oldest first
+   * @returns {RemoteDisk[]}
+   */
+  get disks() {
+    return this.#disks
+  }
+
+  /**
+   * Initializes the VHD chain
+   * @param {Object} options
+   * @param {boolean} options.force
+   * @returns {Promise<void>}
+   */
+  async init(options) {
+    try {
+      await Promise.all(this.#disks.map(disk => disk.init(options)))
+      let parentUuid = ''
+      for (const [index, disk] of this.#disks.entries()) {
+        if (index !== 0) {
+          if (!disk.isDifferencing()) {
+            throw Object.assign(new Error("Can't init vhd directory with non differencing child disks"), {
+              code: 'NOT_SUPPORTED',
+            })
+          }
+          if (disk.getParentUuid() !== parentUuid) {
+            throw Object.assign(new Error("Can't init vhd directory with incorrect parentage"), {
+              code: 'NOT_SUPPORTED',
+            })
+          }
+        }
+
+        parentUuid = disk.getUuid()
+      }
+    } catch (error) {
+      await this.close()
+      throw error
+    }
+  }
+
+  /**
+   * Closes the VHD.
+   * @returns {Promise<void>}
+   */
+  async close() {
+    await Promise.all(this.#disks.map(disk => disk.close()))
+  }
+
+  /**
+   * @returns {number}
+   */
+  getVirtualSize() {
+    return this.#disks[this.#disks.length - 1].getVirtualSize()
+  }
+
+  /**
+   * @returns {number}
+   */
+  getBlockSize() {
+    return this.#disks[this.#disks.length - 1].getBlockSize()
+  }
+
+  /**
+   * @returns {string}
+   */
+  getPath() {
+    return this.#disks[this.#disks.length - 1].getPath()
+  }
+
+  /**
+   * Disk chains return an array of disk paths.
+   *
+   * @returns {string[]}
+   */
+  getPaths() {
+    return this.#disks.map(disk => disk.getPath())
+  }
+
+  /**
+   * @returns {string}
+   */
+  getUuid() {
+    return this.#disks[this.#disks.length - 1].getUuid()
+  }
+
+  /**
+   * @returns {number} getMaxBlockCount
+   */
+  getMaxBlockCount() {
+    return this.#disks[this.#disks.length - 1].getMaxBlockCount()
+  }
+
+  /**
+   * Checks if the VHD contains a specific block.
+   * @param {number} index
+   * @returns {boolean}
+   */
+  hasBlock(index) {
+    for (const disk of this.#disks) {
+      if (disk.hasBlock(index)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  /**
+   * Gets the indexes of all blocks in the VHD.
+   * @returns {Array<number>}
+   */
+  getBlockIndexes() {
+    const indexes = new Set()
+    for (const disk of this.#disks) {
+      for (const index of disk.getBlockIndexes()) {
+        indexes.add(index)
+      }
+    }
+    return [...indexes]
+  }
+
+  /**
+   * Returns the parent non inizialized instance
+   * @returns {RemoteDisk}
+   */
+  instantiateParent() {
+    return this.#disks[0].instantiateParent()
+  }
+
+  /**
+   * Writes a full block into this VHD.
+   * @param {DiskBlock} diskBlock
+   * @return {Promise<number>} blockSize
+   */
+  async writeBlock(diskBlock) {
+    throw new Error(`Can't write blocks into a disk chain`)
+  }
+
+  /**
+   * Reads a specific block from the VHD.
+   * @param {number} index
+   * @returns {Promise<DiskBlock>} diskBlock
+   */
+  async readBlock(index) {
+    const disk = this.newestDiskWith(index)
+    if (disk === undefined) {
+      throw new Error(`Block ${index} not found in chain `)
+    }
+    return disk.readBlock(index)
+  }
+
+  /**
+   * Reads a specific block from the child disk to copy/move it to this disk.
+   * @param {RemoteDisk} childDisk
+   * @param {number} index
+   * @param {boolean} isResumingMerge
+   * @returns {Promise<number>} blockSize
+   */
+  async mergeBlock(childDisk, index, isResumingMerge) {
+    throw new Error(`Can't merge block into a disk chain`)
+  }
+
+  /**
+   * @returns {object}
+   */
+  getMetadata() {
+    return this.#disks[this.#disks.length - 1].getMetadata()
+  }
+
+  /**
+   * @param {RemoteDisk} childDisk
+   * @returns {Promise<void>}
+   */
+  async flushMetadata(childDisk) {
+    throw new Error(`Can't flush metadata on a disk chain`)
+  }
+
+  /**
+   * @param {RemoteDisk} childDisk
+   * @returns {Promise<void>}
+   */
+  mergeMetadata(childDisk) {
+    throw new Error(`Can't merge metadata on a disk chain`)
+  }
+
+  /**
+   * Checks if the VHD is a differencing disk.
+   * @returns {boolean}
+   */
+  isDifferencing() {
+    return this.#disks[0].isDifferencing()
+  }
+
+  /**
+   * Abstract
+   * Rename alias/disk
+   * @param {string} newPath
+   */
+  async rename(newPath) {
+    throw new Error(`Can't rename a disk chain`)
+  }
+
+  /**
+   * Deletes all the disks
+   * @param {Object} options
+   */
+  async unlink({ force = false } = {}) {
+    for (const disk of this.#disks) {
+      await disk.unlink({ force })
+    }
+  }
+
+  /**
+   * Checks the integrity of the disk's external reference (e.g. alias target).
+   * @param {Object} [opts]
+   * @param {boolean} [opts.remove]
+   * @param {Function} [opts.logWarn]
+   * @param {Function} [opts.logInfo]
+   * @returns {Promise<string | undefined>}
+   */
+  async clean(opts = {}) {
+    throw new Error(`Can't clean a disk chain`)
+  }
+
+  /**
+   *
+   * @returns {RemoteDisk|undefined}
+   */
+  getRootDisk() {
+    return this.#disks[0]
+  }
+
+  /**
+   * The disk holding the current content of a block: the newest one having it
+   * @param {number} index
+   * @returns {RemoteDisk|undefined}
+   */
+  newestDiskWith(index) {
+    for (let i = this.#disks.length - 1; i >= 0; i--) {
+      if (this.#disks[i].hasBlock(index)) {
+        return this.#disks[i]
+      }
+    }
+    return undefined
+  }
+}

@xen-orchestra/backup-archive/src/disks/RemoteVhdDisk.mjs

@@ -6,6 +6,7 @@
  * @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
  * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
  * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
+ * @typedef {import('@xen-orchestra/fs').DiskFormat} DiskFormat
  *
 
  */
@@ -17,7 +18,7 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { stringify } from 'uuid'
 import { dirname, join } from 'node:path'
 import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
-import { normalize } from '@xen-orchestra/fs/path'
+import { isInDir, normalize } from '@xen-orchestra/fs/path'
 
 export class RemoteVhdDisk extends RemoteDisk {
   /**
@@ -451,17 +452,18 @@ export class RemoteVhdDisk extends RemoteDisk {
    * sub-format (VHD directory vs plain) and, for directories, the compression codec
    * must match what this remote would write for a new disk.
    * @param {string} parentUuid
+   * @param {{ diskFormat: DiskFormat }} options format of the disks written to the remote
    * @returns {Promise<boolean>}
    */
-  async isMergeableParent(parentUuid) {
-    if (!(await super.isMergeableParent(parentUuid))) {
+  async isMergeableParent(parentUuid, { diskFormat }) {
+    if (!(await super.isMergeableParent(parentUuid, { diskFormat }))) {
       return false
     }
-
-    const useVhdDirectory = this.#handler.getConfig('useVhdDirectory') ?? false
     const compressionType = this.#handler.getConfig('compressionType') ?? 'brotli'
     const isDirectory = await this.isDirectory()
-    return isDirectory ? useVhdDirectory && this.#getCompressionType() === compressionType : !useVhdDirectory
+    return isDirectory
+      ? diskFormat === 'vhdDirectory' && this.#getCompressionType() === compressionType
+      : diskFormat === 'vhd'
   }
 
   /**
@@ -571,18 +573,15 @@ export class RemoteVhdDisk extends RemoteDisk {
    * @returns {Promise<string[]>}
    */
   async listAssociatedFiles(dir) {
-    const prefix = normalize(dir.endsWith('/') ? dir : dir + '/')
-    const isInDir = /** @param {string} p */ p => p === dir || p.startsWith(prefix)
-
     const files = []
-    if (isInDir(this.#path)) {
+    if (isInDir(this.#path, dir)) {
       files.push(this.#path)
     }
 
     if (isVhdAlias(this.#path)) {
       try {
         const resolved = await resolveVhdAlias(this.#handler, this.#path)
-        if (isInDir(resolved)) {
+        if (isInDir(resolved, dir)) {
           files.push(resolved)
         }
       } catch {

@xen-orchestra/backup-archive/src/disks/RemoteVhdDiskChain.mjs

@@ -3,17 +3,11 @@
 /**
  * @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
  * @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
- * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
  */
 
-import { RemoteDisk } from './RemoteDisk.mjs'
-
-export class RemoteVhdDiskChain extends RemoteDisk {
-  /**
-   * @type {RemoteVhdDisk[]}
-   */
-  #disks
+import { RemoteDiskChain } from './RemoteDiskChain.mjs'
 
+export class RemoteVhdDiskChain extends RemoteDiskChain {
   /**
    * @type {number}
    */
@@ -39,55 +33,15 @@ export class RemoteVhdDiskChain extends RemoteDisk {
    * @param {RemoteVhdDisk[]} params.disks
    */
   constructor({ disks }) {
-    super()
-    this.#disks = disks
+    super({ disks })
   }
 
   /**
-   * Initializes the VHD chain
-   * @param {Object} options
-   * @param {boolean} options.force
-   * @returns {Promise<void>}
+   * The VHD disks of the chain, oldest first
+   * @returns {RemoteVhdDisk[]}
    */
-  async init(options) {
-    try {
-      await Promise.all(this.#disks.map(disk => disk.init(options)))
-      let parentUuid = ''
-      for (const [index, disk] of this.#disks.entries()) {
-        if (index !== 0) {
-          if (!disk.isDifferencing()) {
-            throw Object.assign(new Error("Can't init vhd directory with non differencing child disks"), {
-              code: 'NOT_SUPPORTED',
-            })
-          }
-          if (disk.getParentUuid() !== parentUuid) {
-            throw Object.assign(new Error("Can't init vhd directory with incorrect parentage"), {
-              code: 'NOT_SUPPORTED',
-            })
-          }
-        }
-
-        parentUuid = disk.getUuid()
-      }
-    } catch (error) {
-      await this.close()
-      throw error
-    }
-  }
-
-  /**
-   * Closes the VHD.
-   * @returns {Promise<void>}
-   */
-  async close() {
-    await Promise.all(this.#disks.map(disk => disk.close()))
-  }
-
-  /**
-   * @returns {number}
-   */
-  getVirtualSize() {
-    return this.#disks[this.#disks.length - 1].getVirtualSize()
+  get disks() {
+    return /** @type {RemoteVhdDisk[]} */ (super.disks)
   }
 
   /**
@@ -107,36 +61,6 @@ export class RemoteVhdDiskChain extends RemoteDisk {
     )
   }
 
-  /**
-   * @returns {number}
-   */
-  getBlockSize() {
-    return this.#disks[this.#disks.length - 1].getBlockSize()
-  }
-
-  /**
-   * @returns {string}
-   */
-  getPath() {
-    return this.#disks[this.#disks.length - 1].getPath()
-  }
-
-  /**
-   * Disk chains return an array of disk paths.
-   *
-   * @returns {string[]}
-   */
-  getPaths() {
-    return this.#disks.map(disk => disk.getPath())
-  }
-
-  /**
-   * @returns {string}
-   */
-  getUuid() {
-    return this.#disks[this.#disks.length - 1].getUuid()
-  }
-
   /**
    * @returns {Promise<boolean>} canMergeConcurently
    */
@@ -144,148 +68,24 @@ export class RemoteVhdDiskChain extends RemoteDisk {
     return this.isDirectory()
   }
 
-  /**
-   * @returns {number} getMaxBlockCount
-   */
-  getMaxBlockCount() {
-    return this.#disks[this.#disks.length - 1].getMaxBlockCount()
-  }
-
-  /**
-   * Checks if the VHD contains a specific block.
-   * @param {number} index
-   * @returns {boolean}
-   */
-  hasBlock(index) {
-    for (const disk of this.#disks) {
-      if (disk.hasBlock(index)) {
-        return true
-      }
-    }
-
-    return false
-  }
-
-  /**
-   * Gets the indexes of all blocks in the VHD.
-   * @returns {Array<number>}
-   */
-  getBlockIndexes() {
-    const indexes = new Set()
-    for (const disk of this.#disks) {
-      for (const index of disk.getBlockIndexes()) {
-        indexes.add(index)
-      }
-    }
-    return [...indexes]
-  }
-
-  /**
-   * Returns the parent non inizialized instance
-   * @returns {RemoteDisk}
-   */
-  instantiateParent() {
-    return this.#disks[0].instantiateParent()
-  }
-
-  /**
-   * Writes a full block into this VHD.
-   * @param {DiskBlock} diskBlock
-   * @return {Promise<number>} blockSize
-   */
-  async writeBlock(diskBlock) {
-    throw new Error(`Can't write blocks into a disk chain`)
-  }
-
-  /**
-   * Reads a specific block from the VHD.
-   * @param {number} index
-   * @returns {Promise<DiskBlock>} diskBlock
-   */
-  async readBlock(index) {
-    const reversedDisks = this.#disks.slice().reverse()
-    for (const disk of reversedDisks) {
-      if (disk.hasBlock(index)) {
-        return disk.readBlock(index)
-      }
-    }
-    throw new Error(`Block ${index} not found in chain `)
-  }
-
-  /**
-   * Reads a specific block from the child disk to copy/move it to this disk.
-   * @param {RemoteDisk} childDisk
-   * @param {number} index
-   * @param {boolean} isResumingMerge
-   * @returns {Promise<number>} blockSize
-   */
-  async mergeBlock(childDisk, index, isResumingMerge) {
-    throw new Error(`Can't merge block into a disk chain`)
-  }
-
   /**
    * Gets a specific block path from the VHD directory disk.
    * @param {number} index
    * @returns {string} blockPath
    */
   getBlockPath(index) {
-    for (const disk of [...this.#disks].reverse()) {
-      if (disk.hasBlock(index)) {
-        return disk.getBlockPath(index)
-      }
+    const disk = /** @type {RemoteVhdDisk | undefined} */ (this.newestDiskWith(index))
+    if (disk === undefined) {
+      throw new Error(`Block ${index} not found in chain`)
     }
-
-    throw new Error(`Block ${index} not found in chain`)
+    return disk.getBlockPath(index)
   }
 
   /**
    * @returns {VhdFooter}
    */
   getMetadata() {
-    return this.#disks[this.#disks.length - 1].getMetadata()
-  }
-
-  /**
-   * @param {RemoteDisk} childDisk
-   * @returns {Promise<void>}
-   */
-  async flushMetadata(childDisk) {
-    throw new Error(`Can't flush metadata on a disk chain`)
-  }
-
-  /**
-   * @param {RemoteDisk} childDisk
-   * @returns {Promise<void>}
-   */
-  mergeMetadata(childDisk) {
-    throw new Error(`Can't merge metadata on a disk chain`)
-  }
-
-  /**
-   * Checks if the VHD is a differencing disk.
-   * @returns {boolean}
-   */
-  isDifferencing() {
-    return this.#disks[0].isDifferencing()
-  }
-
-  /**
-   * Abstract
-   * Rename alias/disk
-   * @param {string} newPath
-   */
-  async rename(newPath) {
-    throw new Error(`Can't rename a disk chain`)
-  }
-
-  /**
-   * Deletes all the disks
-   * @param {Object} options
-   */
-  async unlink({ force = false } = {}) {
-    for (const disk of this.#disks) {
-      await disk.unlink({ force })
-    }
+    return /** @type {VhdFooter} */ (super.getMetadata())
   }
 
   /**
@@ -293,7 +93,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
    * @returns {Promise<boolean>}
    */
   async isDirectory() {
-    for (const disk of this.#disks) {
+    for (const disk of this.disks) {
       if (!(await disk.isDirectory())) {
         return false
       }
@@ -301,23 +101,11 @@ export class RemoteVhdDiskChain extends RemoteDisk {
     return true
   }
 
-  /**
-   * Checks the integrity of the disk's external reference (e.g. alias target).
-   * @param {Object} [opts]
-   * @param {boolean} [opts.remove]
-   * @param {Function} [opts.logWarn]
-   * @param {Function} [opts.logInfo]
-   * @returns {Promise<string | undefined>}
-   */
-  async clean(opts = {}) {
-    throw new Error(`Can't clean a disk chain`)
-  }
-
   /**
    *
    * @returns {RemoteVhdDisk|undefined}
    */
   getRootDisk() {
-    return this.#disks[0]
+    return /** @type {RemoteVhdDisk|undefined} */ (super.getRootDisk())
   }
 }

@xen-orchestra/backup-archive/src/disks/hashed/BlockAllocationTable.mts

@@ -0,0 +1,119 @@
+import { createLogger } from '@xen-orchestra/log'
+import { asBlockHash, BlockHash, HASH_SIZE } from './hbdPaths.mjs'
+import { readChunkStrict } from '@vates/read-chunk'
+import { Readable } from 'node:stream'
+
+const { warn } = createLogger('xo:backup-archive:bat')
+
+const ZERO_ENTRY = Buffer.alloc(HASH_SIZE)
+
+// a whole number of entries per read: no entry is ever split across two chunks
+const CHUNK_ENTRIES = 2048 // 64 KiB
+
+export class BlockAllocationTable {
+  #bat: Buffer
+  #maxBlockCount: number
+
+  static allocate(maxBlockCount: number): BlockAllocationTable {
+    return new BlockAllocationTable(Buffer.alloc(maxBlockCount * HASH_SIZE), maxBlockCount)
+  }
+
+  resize(maxBlockCount: number): void {
+    const batLength = maxBlockCount * HASH_SIZE
+    if (batLength > this.#bat.length) {
+      const bat = Buffer.alloc(maxBlockCount * HASH_SIZE)
+      this.#bat.copy(bat, 0, 0, Math.min(this.#bat.length, bat.length))
+      this.#bat = bat
+    } else {
+      this.#bat = this.#bat.subarray(0, batLength)
+    }
+    this.#maxBlockCount = maxBlockCount
+  }
+
+  static fromBuffer(buffer: Buffer, maxBlockCount: number, force = false): BlockAllocationTable {
+    const expected = maxBlockCount * HASH_SIZE
+    if (buffer.length !== expected) {
+      const message = `unexpected hashes file size: ${buffer.length} instead of ${expected}`
+      if (force) {
+        warn(message)
+        maxBlockCount = Math.min(maxBlockCount, Math.floor(buffer.length / HASH_SIZE))
+      } else {
+        throw new Error(message)
+      }
+    }
+    return new BlockAllocationTable(buffer, maxBlockCount)
+  }
+
+  private constructor(bat: Buffer, maxBlockCount: number) {
+    this.#bat = bat
+    this.#maxBlockCount = maxBlockCount
+  }
+
+  #entry(index: number): Buffer {
+    if (index < 0 || index >= this.#maxBlockCount) {
+      throw new Error(`block index ${index} out of range [0, ${this.#maxBlockCount})`)
+    }
+    const offset = index * HASH_SIZE
+    return this.#bat.subarray(offset, offset + HASH_SIZE)
+  }
+
+  entryAt(index: number): Buffer {
+    return this.#entry(index)
+  }
+
+  sameAt(index: number, other: BlockAllocationTable, otherIndex: number): boolean {
+    if (index >= this.#maxBlockCount) {
+      return false
+    }
+    return this.entryAt(index).equals(other.entryAt(otherIndex))
+  }
+
+  hashAt(index: number): BlockHash {
+    return this.#entry(index).toString('hex') as BlockHash
+  }
+
+  set(index: number, hash: BlockHash) {
+    if (this.#entry(index).write(asBlockHash(hash), 'hex') !== HASH_SIZE) {
+      throw new Error(`incomplete write of block hash ${hash}`)
+    }
+  }
+
+  isEmpty(index: number): boolean {
+    if (index >= this.#maxBlockCount) {
+      return true
+    }
+    const offset = index * HASH_SIZE
+    return this.#bat.compare(ZERO_ENTRY, 0, HASH_SIZE, offset, offset + HASH_SIZE) === 0
+  }
+
+  countAllocated(): number {
+    let total = 0
+    for (let idx = 0; idx < this.#maxBlockCount; idx++) {
+      if (!this.isEmpty(idx)) total++
+    }
+    return total
+  }
+
+  indexes(): Array<number> {
+    const res = new Array<number>()
+    for (let idx = 0; idx < this.#maxBlockCount; idx++) {
+      if (!this.isEmpty(idx)) res.push(idx)
+    }
+    return res
+  }
+
+  toBuffer(): Buffer {
+    return this.#bat
+  }
+
+  static async *readChunks(
+    stream: Readable,
+    blockCount: number
+  ): AsyncGenerator<{ firstIndex: number; bat: BlockAllocationTable }> {
+    for (let firstIndex = 0; firstIndex < blockCount; firstIndex += CHUNK_ENTRIES) {
+      const count = Math.min(CHUNK_ENTRIES, blockCount - firstIndex)
+      const chunk = (await readChunkStrict(stream, count * HASH_SIZE)) as Buffer
+      yield { firstIndex, bat: new BlockAllocationTable(chunk, count) }
+    }
+  }
+}

@xen-orchestra/backup-archive/src/disks/hashed/HashedDisk.mts

@@ -0,0 +1,3 @@
+import { RemoteDisk } from '../RemoteDisk.mjs'
+
+export class HashedDisk extends RemoteDisk {}

@xen-orchestra/backup-archive/src/disks/hashed/HashedDiskChain.mts

@@ -0,0 +1,50 @@
+import { RemoteDiskChain } from '../RemoteDiskChain.mjs'
+// type only: HashedDiskDeduplicated imports this class at runtime
+import type { HashedDiskDeduplicated } from './HashedDiskDeduplicated.mjs'
+import type { BlockHash } from './hbdPaths.mjs'
+
+/**
+ * A chain of hbd disks, oldest first, read as a single disk.
+ *
+ * The child of a hbd merge is always one of these (see openDiskChain): it
+ * exposes what the merge needs from the newest disk holding each block.
+ */
+export class HashedDiskChain extends RemoteDiskChain {
+  constructor({ disks }: { disks: HashedDiskDeduplicated[] }) {
+    super({ disks })
+  }
+
+  get disks(): HashedDiskDeduplicated[] {
+    return super.disks as HashedDiskDeduplicated[]
+  }
+
+  getBlockHashAt(index: number): BlockHash {
+    return this.#newestDiskWith(index).getBlockHashAt(index)
+  }
+
+  getBlockPath(index: number): string {
+    return this.#newestDiskWith(index).getBlockPath(index)
+  }
+
+  /** block files are independent, nothing is shared inside a single disk */
+  async canMergeConcurently(): Promise<boolean> {
+    return true
+  }
+
+  /** Virtual usage of each disk, as HashedDiskDeduplicated.getSizeOnDisk */
+  getSizeOnDisk(): number {
+    return this.disks.reduce((size, disk) => size + disk.getSizeOnDisk(), 0)
+  }
+
+  getRootDisk(): HashedDiskDeduplicated | undefined {
+    return this.disks[0]
+  }
+
+  #newestDiskWith(index: number): HashedDiskDeduplicated {
+    const disk = this.newestDiskWith(index)
+    if (disk === undefined) {
+      throw new Error(`Block ${index} not found in chain`)
+    }
+    return disk as HashedDiskDeduplicated
+  }
+}

@xen-orchestra/backup-archive/src/disks/hashed/HashedDiskChain.test.mjs

@@ -0,0 +1,241 @@
+import { test } from 'node:test'
+import { strict as assert } from 'node:assert'
+
+import tmp from 'tmp'
+import * as uuid from 'uuid'
+import { getHandler } from '@xen-orchestra/fs'
+import { normalize } from '@xen-orchestra/fs/path'
+import { pFromCallback } from 'promise-toolbox'
+import { rimraf } from 'rimraf'
+
+import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
+import { HashedDiskChain, HashedDiskDeduplicated, HbdFileError } from '@xen-orchestra/backup-archive/disks/hashed'
+
+const { beforeEach, afterEach, describe } = test
+
+const BLOCK_SIZE = 4096
+const BLOCK_COUNT = 4
+
+const block = byte => Buffer.alloc(BLOCK_SIZE, byte)
+
+let tempDir, handler, diskDir
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  diskDir = `xo-vm-backups/VMUUID/vdis/job/${uuid.v4()}`
+})
+
+afterEach(async () => {
+  await handler.forget()
+  await rimraf(tempDir)
+})
+
+const diskPath = n => `${diskDir}/2026081${n}T120000000Z.hbd`
+
+/**
+ * Writes a chain of hbd disks, oldest first: `contents[n]` maps index -> fill byte of disk n
+ * @returns the paths and uuids, oldest first
+ */
+async function writeChain(contents) {
+  const disks = []
+  for (const [n, content] of contents.entries()) {
+    const path = diskPath(n)
+    const diskUuid = uuid.v4()
+    const parent = disks[n - 1]
+    const disk = await HashedDiskDeduplicated.create({
+      handler,
+      path,
+      virtualSize: BLOCK_SIZE * BLOCK_COUNT,
+      blockSize: BLOCK_SIZE,
+      uuid: diskUuid,
+      parentUuid: parent?.uuid,
+      parentPath: parent === undefined ? undefined : `./2026081${n - 1}T120000000Z.hbd`,
+      dedupType: 'PER_BACKUP_REPOSITORY',
+    })
+    for (const [index, byte] of Object.entries(content)) {
+      await disk.writeBlock({ index: Number(index), data: block(byte) })
+    }
+    await disk.close()
+    disks.push({ path, uuid: diskUuid })
+  }
+  return disks
+}
+
+async function openChain(paths) {
+  const disks = []
+  for (const path of paths) {
+    const disk = new HashedDiskDeduplicated({ handler, path })
+    await disk.init()
+    disks.push(disk)
+  }
+  return new HashedDiskChain({ disks })
+}
+
+// 3 disks: index 0 only in the base, 1 rewritten by each disk, 2 only in the middle one, 3 nowhere
+const CHAIN_OF_3 = [{ 0: 0x10, 1: 0x11 }, { 1: 0x21, 2: 0x22 }, { 1: 0x31 }]
+
+describe('HashedDiskChain', () => {
+  test('reads each block from the newest disk holding it', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+    const chain = await openChain(written.map(_ => _.path))
+
+    assert.deepEqual([...chain.getBlockIndexes()].sort(), [0, 1, 2])
+    assert.equal(chain.hasBlock(3), false)
+    for (const [index, byte] of [
+      [0, 0x10],
+      [1, 0x31],
+      [2, 0x22],
+    ]) {
+      assert.ok((await chain.readBlock(index)).data.equals(block(byte)), `block ${index}`)
+    }
+    await assert.rejects(() => chain.readBlock(3), /not found in chain/)
+  })
+
+  test('getBlockHashAt and getBlockPath come from the newest disk holding the block', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+    const chain = await openChain(written.map(_ => _.path))
+    const [base, middle, top] = chain.disks
+
+    assert.equal(chain.getBlockHashAt(1), top.getBlockHashAt(1))
+    assert.equal(chain.getBlockPath(1), top.getBlockPath(1))
+    assert.equal(chain.getBlockHashAt(2), middle.getBlockHashAt(2))
+    assert.equal(chain.getBlockPath(2), middle.getBlockPath(2))
+    assert.equal(chain.getBlockPath(0), base.getBlockPath(0))
+    // the path is a real file, holding the block of that disk
+    await handler.readFile(chain.getBlockPath(2))
+
+    assert.throws(() => chain.getBlockHashAt(3), /not found in chain/)
+    assert.throws(() => chain.getBlockPath(3), /not found in chain/)
+  })
+
+  test('exposes the newest disk identity and the summed size', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+    const chain = await openChain(written.map(_ => _.path))
+
+    assert.equal(chain.getUuid(), written[2].uuid)
+    assert.equal(chain.getPath(), normalize(written[2].path))
+    assert.deepEqual(
+      chain.getPaths(),
+      written.map(_ => normalize(_.path))
+    )
+    assert.equal(chain.getRootDisk(), chain.disks[0])
+    assert.equal(chain.getSizeOnDisk(), (2 + 2 + 1) * BLOCK_SIZE)
+    assert.equal(await chain.canMergeConcurently(), true)
+  })
+
+  test('init refuses a broken parentage, and closes the disks', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+    // the middle disk is skipped: the top one is not a child of the base
+    const disks = [written[0], written[2]].map(({ path }) => new HashedDiskDeduplicated({ handler, path }))
+    const closed = []
+    for (const disk of disks) {
+      disk.close = async () => closed.push(disk)
+    }
+
+    await assert.rejects(() => new HashedDiskChain({ disks }).init(), { code: 'NOT_SUPPORTED' })
+    assert.equal(closed.length, 2)
+  })
+})
+
+describe('merging a chain child into a hbd', () => {
+  async function openParentAndChildChain() {
+    const written = await writeChain(CHAIN_OF_3)
+    const parent = new HashedDiskDeduplicated({ handler, path: written[0].path })
+    await parent.init()
+    const child = await openChain(written.slice(1).map(_ => _.path))
+    return { written, parent, child }
+  }
+
+  test('mergeBlock links the block of the newest child disk, and skips identical or absent blocks', async () => {
+    const { parent, child } = await openParentAndChildChain()
+
+    assert.equal(await parent.mergeBlock(child, 1, false), BLOCK_SIZE)
+    assert.equal(parent.getBlockHashAt(1), child.getBlockHashAt(1))
+    assert.ok((await parent.readBlock(1)).data.equals(block(0x31)))
+
+    // already holding the child's content
+    assert.equal(await parent.mergeBlock(child, 1, false), 0)
+    // nothing in the child at this index
+    assert.equal(await parent.mergeBlock(child, 3, false), 0)
+  })
+
+  test('flushMetadata(child) re-syncs every index of the chain', async () => {
+    const { parent, child } = await openParentAndChildChain()
+
+    await parent.flushMetadata(child)
+
+    for (const [index, byte] of [
+      [0, 0x10],
+      [1, 0x31],
+      [2, 0x22],
+    ]) {
+      assert.ok((await parent.readBlock(index)).data.equals(block(byte)), `block ${index}`)
+    }
+  })
+
+  test('mergeMetadata(child) takes the identity of the newest disk of the chain', async () => {
+    const { written, parent, child } = await openParentAndChildChain()
+
+    await parent.mergeMetadata(child)
+
+    assert.equal(parent.getUuid(), written[2].uuid)
+  })
+
+  test('refuses a child of another type, and never skips the re-sync silently', async () => {
+    const { parent } = await openParentAndChildChain()
+
+    await assert.rejects(() => parent.flushMetadata({}), /can't merge different disk types/)
+    await assert.rejects(() => parent.mergeMetadata({}), /can't merge different disk types/)
+  })
+})
+
+describe('openDiskChain on hbd disks', () => {
+  test('returns a HashedDiskChain of the whole chain', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+
+    const chain = await openDiskChain({ handler, path: written[2].path })
+    try {
+      assert.ok(chain instanceof HashedDiskChain)
+      assert.deepEqual(
+        chain.getPaths(),
+        written.map(_ => normalize(_.path))
+      )
+    } finally {
+      await chain.close()
+    }
+  })
+
+  test('stops at `until`, like a merge opens its child', async () => {
+    const written = await writeChain(CHAIN_OF_3)
+
+    const chain = await openDiskChain({ handler, path: written[2].path, until: normalize(written[0].path) })
+    try {
+      assert.ok(chain instanceof HashedDiskChain)
+      assert.deepEqual(
+        chain.getPaths(),
+        written.slice(1).map(_ => normalize(_.path))
+      )
+    } finally {
+      await chain.close()
+    }
+  })
+
+  test('refuses a hbd whose parent is not a hbd', async () => {
+    const written = await writeChain([{ 0: 0x10 }])
+    // points the disk at a VHD parent: it is opened as a hbd, and can't be read as one
+    const metadata = JSON.parse(await handler.readFile(written[0].path))
+    await handler.writeFile(`${diskDir}/parent.vhd`, Buffer.alloc(1024))
+    await handler.writeFile(
+      written[0].path,
+      JSON.stringify({ ...metadata, parentUuid: uuid.v4(), parentPath: './parent.vhd' }),
+      { flags: 'w' }
+    )
+
+    await assert.rejects(
+      () => openDiskChain({ handler, path: written[0].path }),
+      error => error instanceof HbdFileError && error.path.endsWith('parent.vhd')
+    )
+  })
+})

@xen-orchestra/backup-archive/src/disks/hashed/HashedDiskDeduplicated.integ.mjs

@@ -0,0 +1,305 @@
+import { test } from 'node:test'
+import { strict as assert } from 'node:assert'
+import { stat } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { rimraf } from 'rimraf'
+
+import { MergeRemoteDisk } from '@xen-orchestra/backup-archive/disks'
+import { HashedDiskDeduplicated } from '@xen-orchestra/backup-archive/disks/hashed'
+// not exported by the package: built by the `test-integration` script
+import { RemoteDiskLineage } from '../../../dist/RemoteDiskLineage.mjs'
+
+const BLOCK_SIZE = 2 * 1024 * 1024
+const BLOCK_COUNT = 100
+const UNIQUE_PAYLOADS = 70
+
+async function listFiles(handler, dir) {
+  const found = []
+  for (const entry of await handler.list(dir, { prependDir: true })) {
+    try {
+      found.push(...(await listFiles(handler, entry)))
+    } catch (error) {
+      if (error.code !== 'ENOTDIR') {
+        throw error
+      }
+      found.push(entry)
+    }
+  }
+  return found
+}
+
+// index i holds payload (i % UNIQUE_PAYLOADS), so the last 30 blocks repeat earlier content
+const payloadOf = index => Buffer.alloc(BLOCK_SIZE, (index % UNIQUE_PAYLOADS) % 256)
+
+test('a full disk round trips, and duplicate payloads are stored once', async () => {
+  const tempDir = await pFromCallback(cb => tmp.dir(cb))
+  const handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+
+  try {
+    const path = 'xo-vm-backups/VMUUID/vdis/job/vdi/20260814T120000000Z.hbd'
+    const disk = await HashedDiskDeduplicated.create({
+      handler,
+      path,
+      virtualSize: BLOCK_SIZE * BLOCK_COUNT,
+      blockSize: BLOCK_SIZE,
+      uuid: '7d1e4c92-3f60-4a8b-b5c7-1e9f08d2a643',
+    })
+
+    for (let index = 0; index < BLOCK_COUNT; index++) {
+      assert.equal(await disk.writeBlock({ index, data: payloadOf(index) }), BLOCK_SIZE)
+    }
+
+    assert.equal(disk.getBlockIndexes().length, BLOCK_COUNT)
+    assert.equal(disk.getSizeOnDisk(), BLOCK_COUNT * BLOCK_SIZE)
+
+    // the repeated payloads share a hash, so they share a file
+    assert.equal(disk.getBlockHashAt(0), disk.getBlockHashAt(UNIQUE_PAYLOADS))
+
+    await disk.close()
+
+    // reopen from scratch: nothing may be kept in memory
+    const reopened = new HashedDiskDeduplicated({ handler, path })
+    await reopened.init()
+
+    assert.equal(reopened.getBlockIndexes().length, BLOCK_COUNT)
+    for (let index = 0; index < BLOCK_COUNT; index++) {
+      const { data } = await reopened.readBlock(index)
+      assert.ok(data.equals(payloadOf(index)), `block ${index} differs`)
+    }
+
+    const blockFiles = (await listFiles(handler, 'xo-vm-backups')).filter(file => file.includes('/blocks/'))
+    assert.equal(blockFiles.length, UNIQUE_PAYLOADS, `${BLOCK_COUNT} blocks stored as ${UNIQUE_PAYLOADS} files`)
+
+    await reopened.unlink()
+    assert.deepEqual(await listFiles(handler, 'xo-vm-backups'), [])
+  } finally {
+    await handler.forget()
+    await rimraf(tempDir)
+  }
+})
+
+// ---------------------------------------------------------------- merge
+
+const STORE = 'xo-block-store'
+const VDI_DIR = 'xo-vm-backups/VMUUID/vdis/job/vdi'
+const PARENT = { path: `${VDI_DIR}/20260814T120000000Z.hbd`, uuid: '0b3f2c7e-8a41-4d5f-9c26-7e1a5b8d3f04' }
+const CHILD = { path: `${VDI_DIR}/20260815T120000000Z.hbd`, uuid: 'c4e9a1d2-6b7f-4e83-a5c0-2d8f9b1e6a37' }
+const MERGE_BLOCKS = 5
+// 3 changed, 2 shared with the parent
+const parentContent = index => Buffer.alloc(BLOCK_SIZE, index + 1)
+const childContent = index => (index % 2 === 0 ? Buffer.alloc(BLOCK_SIZE, 0x80 + index) : parentContent(index))
+
+async function withRemote(fn) {
+  const tempDir = await pFromCallback(cb => tmp.dir(cb))
+  const handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  try {
+    await fn(handler, tempDir)
+  } finally {
+    await handler.forget()
+    await rimraf(tempDir)
+  }
+}
+
+async function createChain(handler) {
+  const common = {
+    handler,
+    virtualSize: BLOCK_SIZE * MERGE_BLOCKS,
+    blockSize: BLOCK_SIZE,
+    dedupType: 'PER_BACKUP_REPOSITORY',
+    blockStorePath: STORE,
+  }
+  const parent = await HashedDiskDeduplicated.create({ ...common, ...PARENT })
+  const child = await HashedDiskDeduplicated.create({
+    ...common,
+    ...CHILD,
+    parentUuid: PARENT.uuid,
+    parentPath: './20260814T120000000Z.hbd',
+  })
+  for (let index = 0; index < MERGE_BLOCKS; index++) {
+    await parent.writeBlock({ index, data: parentContent(index) })
+    await child.writeBlock({ index, data: childContent(index) })
+  }
+  await parent.close()
+  await child.close()
+}
+
+const openDisk = async (handler, path) => {
+  const disk = new HashedDiskDeduplicated({ handler, path, blockStorePath: STORE })
+  await disk.init()
+  return disk
+}
+
+// what is left after a merge: the child content, under the child name and uuid
+async function assertMerged(handler, tempDir) {
+  const merged = await openDisk(handler, CHILD.path)
+  assert.equal(merged.getUuid(), CHILD.uuid)
+  assert.equal(merged.isDifferencing(), false, 'the merged disk keeps the parent own (absent) parent')
+  for (let index = 0; index < MERGE_BLOCKS; index++) {
+    assert.ok((await merged.readBlock(index)).data.equals(childContent(index)), `block ${index} differs`)
+  }
+
+  await assert.rejects(() => handler.readFile(PARENT.path), { code: 'ENOENT' })
+  const files = await listFiles(handler, VDI_DIR)
+  assert.equal(
+    files.some(file => file.includes(`/data/${CHILD.uuid}/`)),
+    false,
+    'the child data dir is gone'
+  )
+  assert.equal(
+    files.some(file => file.endsWith('.merge.json')),
+    false,
+    'no merge state left'
+  )
+
+  // one store file per block still referenced, each held by the store copy and the merged disk only
+  const storeFiles = await listFiles(handler, STORE)
+  assert.equal(storeFiles.length, MERGE_BLOCKS)
+  for (const storeFile of storeFiles) {
+    assert.equal((await stat(join(tempDir, storeFile))).nlink, 2, storeFile)
+  }
+}
+
+test('MergeRemoteDisk merges a hashed child into its parent', async () => {
+  await withRemote(async (handler, tempDir) => {
+    await createChain(handler)
+
+    const merger = new MergeRemoteDisk(handler, { removeUnused: true })
+    await merger.merge(await openDisk(handler, PARENT.path), await openDisk(handler, CHILD.path))
+
+    await assertMerged(handler, tempDir)
+  })
+})
+
+test('MergeRemoteDisk resumes a hashed merge interrupted before the parent was flushed', async () => {
+  await withRemote(async (handler, tempDir) => {
+    await createChain(handler)
+
+    // first run: dies on the 4th block, like a killed process, so nothing is flushed
+    const parent = await openDisk(handler, PARENT.path)
+    const mergeBlock = parent.mergeBlock.bind(parent)
+    let calls = 0
+    parent.mergeBlock = (...args) => {
+      if (++calls === 4) {
+        throw new Error('crash')
+      }
+      return mergeBlock(...args)
+    }
+    parent.close = async () => {}
+    const firstRun = new MergeRemoteDisk(handler, { removeUnused: true, mergeBlockConcurrency: 1, writeStateDelay: 0 })
+    const child = await openDisk(handler, CHILD.path)
+    await assert.rejects(() => firstRun.merge(parent, child), /crash/)
+    assert.ok(
+      (await listFiles(handler, VDI_DIR)).some(file => file.endsWith('.merge.json')),
+      'the merge state is kept'
+    )
+
+    // second run: fresh disks, restarts from the saved block and skips the ones before
+    const resumed = new MergeRemoteDisk(handler, { removeUnused: true })
+    await resumed.merge(await openDisk(handler, PARENT.path), await openDisk(handler, CHILD.path))
+
+    await assertMerged(handler, tempDir)
+  })
+})
+
+// ---------------------------------------------------------------- merge through RemoteDiskLineage
+
+// what a backup cleanup does: the lineage opens the merge child with openDiskChain, so the
+// child is a HashedDiskChain, never a bare disk
+const GRANDCHILD = { path: `${VDI_DIR}/20260816T120000000Z.hbd`, uuid: 'e2a7c5b1-9d34-4f6e-8b02-5c1d7a9e3f48' }
+const grandchildContent = index => (index === 1 ? Buffer.alloc(BLOCK_SIZE, 0xc1) : childContent(index))
+
+async function addGrandchild(handler) {
+  const grandchild = await HashedDiskDeduplicated.create({
+    handler,
+    ...GRANDCHILD,
+    virtualSize: BLOCK_SIZE * MERGE_BLOCKS,
+    blockSize: BLOCK_SIZE,
+    dedupType: 'PER_BACKUP_REPOSITORY',
+    blockStorePath: STORE,
+    parentUuid: CHILD.uuid,
+    parentPath: './20260815T120000000Z.hbd',
+  })
+  await grandchild.writeBlock({ index: 1, data: grandchildContent(1) })
+  await grandchild.close()
+}
+
+const LINEAGE_OPTS = { remove: true, merge: true, logInfo: () => {}, logWarn: () => {} }
+
+// only `activePath` is still referenced by a backup: every older disk is merged into it
+async function cleanLineage(handler, activePath, opts = {}) {
+  const lineage = new RemoteDiskLineage(handler, VDI_DIR, { ...LINEAGE_OPTS, ...opts })
+  await lineage.init()
+  lineage.addActiveDiskPath(activePath)
+  return lineage.clean()
+}
+
+test('RemoteDiskLineage merges a chain of 2 hbd disks', async () => {
+  await withRemote(async (handler, tempDir) => {
+    await createChain(handler)
+
+    const { merge } = await cleanLineage(handler, CHILD.path)
+
+    assert.equal(merge, true)
+    await assertMerged(handler, tempDir)
+  })
+})
+
+test('RemoteDiskLineage merges a chain of 3 hbd disks', async () => {
+  await withRemote(async handler => {
+    await createChain(handler)
+    await addGrandchild(handler)
+
+    await cleanLineage(handler, GRANDCHILD.path)
+
+    assert.deepEqual(
+      (await handler.list(VDI_DIR)).filter(name => name.endsWith('.hbd')),
+      ['20260816T120000000Z.hbd'],
+      'only the active disk is left'
+    )
+    const merged = await openDisk(handler, GRANDCHILD.path)
+    assert.equal(merged.getUuid(), GRANDCHILD.uuid)
+    assert.equal(merged.isDifferencing(), false)
+    for (let index = 0; index < MERGE_BLOCKS; index++) {
+      assert.ok((await merged.readBlock(index)).data.equals(grandchildContent(index)), `block ${index} differs`)
+    }
+    // the store only holds what the merged disk still references
+    const referenced = new Set(merged.getBlockIndexes().map(index => merged.getBlockHashAt(index)))
+    assert.equal((await listFiles(handler, STORE)).length, referenced.size)
+  })
+})
+
+test('RemoteDiskLineage resumes an interrupted merge of a hbd chain', async () => {
+  await withRemote(async (handler, tempDir) => {
+    await createChain(handler)
+
+    // first run: dies on the 4th block, like a killed process
+    const { mergeBlock } = HashedDiskDeduplicated.prototype
+    let calls = 0
+    HashedDiskDeduplicated.prototype.mergeBlock = function (...args) {
+      if (++calls === 4) {
+        throw new Error('crash')
+      }
+      return mergeBlock.apply(this, args)
+    }
+    try {
+      await assert.rejects(() => cleanLineage(handler, CHILD.path, { mergeBlockConcurrency: 1 }), /crash/)
+    } finally {
+      HashedDiskDeduplicated.prototype.mergeBlock = mergeBlock
+    }
+    assert.ok(
+      (await listFiles(handler, VDI_DIR)).some(file => file.endsWith('.merge.json')),
+      'the merge state is kept'
+    )
+
+    // second run: a fresh lineage finds the merge state and resumes it
+    await cleanLineage(handler, CHILD.path)
+
+    await assertMerged(handler, tempDir)
+  })
+})

@xen-orchestra/backup-archive/src/disks/hashed/HashedDiskDeduplicated.mts

@@ -0,0 +1,810 @@
+import type { DiskBlock } from '@xen-orchestra/disk-transform'
+import { type RemoteHandlerAbstract, type DiskFormat } from '@xen-orchestra/fs'
+import { basename, dirname, join } from 'node:path'
+import { Readable } from 'node:stream'
+import { isInDir, normalize } from '@xen-orchestra/fs/path'
+import pRetry from 'promise-toolbox/retry'
+import { createLogger } from '@xen-orchestra/log'
+
+import { HashedDisk } from './HashedDisk.mjs'
+import { HashedDiskChain } from './HashedDiskChain.mjs'
+import { BlockAllocationTable } from './BlockAllocationTable.mjs'
+import {
+  blockRelPath,
+  checkVersion,
+  dataDirName,
+  decodeBlock,
+  encodeBlock,
+  hashesFileName,
+  HbdFileError,
+  sha256hex,
+  VERSION,
+  type DedupType,
+  type BlockHash,
+  type HashedDiskMetadata,
+  HashEntrySet,
+  HBD_HEADER_SIZE,
+  BLOCK_STORE_PATH,
+  parseBlockHeader,
+  CODEC_RAW,
+  setVerifiedAt,
+  HBD_MAGIC,
+} from './hbdPaths.mjs'
+import { randomUUID } from 'node:crypto'
+
+const { warn, debug } = createLogger('xo:backup-archive:hbd')
+
+const LEFTOVER_SAMPLE = 5
+
+/**
+ * What a hbd can be merged with: a single hbd, or a chain of them
+ */
+type MergeChild = HashedDiskDeduplicated | HashedDiskChain
+
+function assertMergeChild(childDisk: unknown): asserts childDisk is MergeChild {
+  if (!(childDisk instanceof HashedDiskDeduplicated || childDisk instanceof HashedDiskChain)) {
+    throw new Error(`can't merge different disk types`)
+  }
+}
+
+/**
+ * Content addressed disk: block index -> SHA-256 of the block payload, kept in a
+ * flat binary BAT, and one file per unique payload named after its hash.
+ */
+export class HashedDiskDeduplicated extends HashedDisk {
+  #handler: RemoteHandlerAbstract
+  #path: string
+  #metadata: HashedDiskMetadata | undefined
+  #bat: BlockAllocationTable | undefined
+  #dataDir: string | undefined
+  #blocksDir: string | undefined
+  #blockStorePath: string
+  #dirty = false
+  #oldBlockCount = 0
+  #scrubEnabled: boolean
+  #scrubMaxAgeMs: number
+  #blocksScrubbed = 0
+  #blocksCorrupted = 0
+
+  constructor({
+    handler,
+    path,
+    blockStorePath = BLOCK_STORE_PATH,
+    scrubEnabled = false,
+    scrubMaxAgeMs = 30 * 86400e3, // 30 days
+  }: {
+    handler: RemoteHandlerAbstract
+    path: string
+    blockStorePath?: string
+    scrubEnabled?: boolean
+    scrubMaxAgeMs?: number
+  }) {
+    super()
+    this.#handler = handler
+    // normalized once here so every path this disk hands out or derives has the
+    // same shape: callers match them against paths listed from the handler, and
+    // an unnormalized one silently fails to compare equal
+    this.#path = normalize(path)
+    this.#blockStorePath = blockStorePath
+
+    this.#scrubEnabled = scrubEnabled && !this.#handler.isEncrypted
+    if (scrubEnabled && !this.#scrubEnabled) {
+      warn("Scrubbing can't be enabled on a encrypted remote", { path: this.#path })
+    }
+    this.#scrubMaxAgeMs = scrubMaxAgeMs
+  }
+
+  /**
+   * New, empty disk on the remote and returns it, opened.
+   */
+  static async create({
+    handler,
+    path,
+    virtualSize,
+    blockSize,
+    uuid,
+    parentUuid,
+    parentPath,
+    dedupType = 'PER_DISK',
+    blockStorePath = BLOCK_STORE_PATH,
+    scrubEnabled,
+    scrubMaxAgeMs,
+  }: {
+    handler: RemoteHandlerAbstract
+    path: string
+    virtualSize: number
+    blockSize: number
+    uuid: string
+    parentUuid?: string
+    parentPath?: string
+    dedupType?: DedupType
+    blockStorePath?: string
+    scrubEnabled?: boolean
+    scrubMaxAgeMs?: number
+  }): Promise<HashedDiskDeduplicated> {
+    const dataDir = dataDirName(uuid)
+    const hashesPath = join(dataDir, hashesFileName(new Date()))
+
+    const metadata = {
+      version: VERSION,
+      virtualSize,
+      blockSize,
+      uuid,
+      parentUuid,
+      parentPath,
+      dedupType,
+      localBlocksPath: `${dataDir}/blocks/`,
+      hashesPath,
+    } satisfies HashedDiskMetadata
+
+    const bat = BlockAllocationTable.allocate(Math.ceil(virtualSize / blockSize))
+
+    const disk = new HashedDiskDeduplicated({ handler, path, blockStorePath, scrubEnabled, scrubMaxAgeMs })
+    await handler.outputFile(disk.#resolve(hashesPath), bat.toBuffer(), { flags: 'wx' })
+    await handler.outputFile(path, JSON.stringify(metadata), { flags: 'wx' })
+
+    await disk.init()
+    return disk
+  }
+
+  // ---------------------------------------------------------------- getters
+
+  get #diskDir(): string {
+    return dirname(this.#path)
+  }
+
+  get #loadedMetadata(): HashedDiskMetadata {
+    if (this.#metadata === undefined) {
+      throw new Error(`can't use a HashedDiskDeduplicated before init`)
+    }
+    return this.#metadata
+  }
+
+  get #loadedBat(): BlockAllocationTable {
+    if (this.#bat === undefined) {
+      throw new Error(`can't use a HashedDiskDeduplicated before init`)
+    }
+    return this.#bat
+  }
+
+  /**
+   * holds the blocks and every hashes file, current and orphaned
+   */
+  get #loadedDataDir(): string {
+    if (this.#dataDir === undefined) {
+      throw new Error(`can't use a HashedDiskDeduplicated before init`)
+    }
+    return this.#dataDir
+  }
+
+  getVirtualSize(): number {
+    return this.#loadedMetadata.virtualSize
+  }
+
+  getBlockSize(): number {
+    return this.#loadedMetadata.blockSize
+  }
+
+  /**
+   * Virtual usage: what the guest sees as allocated, not the deduplicated footprint
+   */
+  getSizeOnDisk(): number {
+    return this.#loadedBat.countAllocated() * this.getBlockSize()
+  }
+
+  getPath(): string {
+    return this.#path
+  }
+
+  getPaths(): Array<string> {
+    return [this.#path]
+  }
+
+  getUuid(): string {
+    return this.#loadedMetadata.uuid
+  }
+
+  getParentUuid(): string {
+    const { parentUuid } = this.#loadedMetadata
+    if (parentUuid === undefined) {
+      throw new Error(`disk ${this.#path} has no parent`)
+    }
+    return parentUuid
+  }
+
+  getParentPath(): string {
+    const { parentPath } = this.#loadedMetadata
+    if (parentPath === undefined) {
+      throw new Error(`disk ${this.#path} has no parent`)
+    }
+    return this.#resolve(parentPath)
+  }
+
+  getMetadata(): HashedDiskMetadata {
+    return { ...this.#loadedMetadata }
+  }
+
+  isDifferencing(): boolean {
+    return this.#loadedMetadata.parentUuid !== undefined
+  }
+
+  /** block files are independent, nothing is shared inside a single disk */
+  async canMergeConcurently(): Promise<boolean> {
+    return true
+  }
+
+  hasBlock(index: number): boolean {
+    return !this.#loadedBat.isEmpty(index)
+  }
+
+  getBlockIndexes(): Array<number> {
+    return this.#loadedBat.indexes()
+  }
+
+  getBlockHashAt(index: number): BlockHash {
+    return this.#loadedBat.hashAt(index)
+  }
+
+  /**
+   * Path of the file holding the block at this index, inside this disk
+   */
+  getBlockPath(index: number): string {
+    if (!this.hasBlock(index)) {
+      throw new Error(`no block at index ${index} of ${this.#path}`)
+    }
+    return this.#blockPath(this.getBlockHashAt(index))
+  }
+
+  /** store blocks fully reread and checked since this disk was opened */
+  getBlocksScrubbed(): number {
+    return this.#blocksScrubbed
+  }
+
+  /** store blocks found corrupted, and rewritten, since this disk was opened */
+  getBlocksCorrupted(): number {
+    return this.#blocksCorrupted
+  }
+
+  async isMergeableParent(parentUuid: string, { diskFormat }: { diskFormat: DiskFormat }): Promise<boolean> {
+    return (await super.isMergeableParent(parentUuid, { diskFormat })) && diskFormat === 'hbd'
+  }
+
+  /**
+   * The parent of a hbd is always opened as a hbd: a chain never mixes formats,
+   * a parent of another format fails at init
+   */
+  instantiateParent(): HashedDiskDeduplicated {
+    return new HashedDiskDeduplicated({
+      handler: this.#handler,
+      path: this.getParentPath(),
+      blockStorePath: this.#blockStorePath,
+      scrubEnabled: this.#scrubEnabled,
+      scrubMaxAgeMs: this.#scrubMaxAgeMs,
+    })
+  }
+
+  // ---------------------------------------------------------------- public
+
+  /**
+   * @param options.force to force read a hashes file whose size disagrees with
+   * virtualSize / blockSize, instead of refusing to open the disk (useful for first tests)
+   */
+  async init(options: { force?: boolean } = {}): Promise<void> {
+    if (this.#metadata !== undefined) {
+      return
+    }
+
+    let metadata: HashedDiskMetadata
+    let dataDir: string
+    let hashesPath: string
+    let blocksDir: string
+    try {
+      metadata = JSON.parse((await this.#handler.readFile(this.#path)).toString())
+      checkVersion(metadata.version)
+
+      const { blockSize, virtualSize } = metadata
+      if (!Number.isInteger(blockSize) || blockSize <= 0) {
+        throw new Error(`invalid blockSize ${blockSize}`)
+      }
+      if (!Number.isInteger(virtualSize) || virtualSize < 0) {
+        throw new Error(`invalid virtualSize ${virtualSize}`)
+      }
+
+      // named after the uuid at creation, not the current one: a merge gives
+      // the disk its child's uuid but leaves the directory where it is
+      dataDir = this.#resolve(dataDirName(basename(dirname(metadata.localBlocksPath))))
+      hashesPath = this.#resolve(metadata.hashesPath, dataDir)
+      blocksDir = this.#resolve(metadata.localBlocksPath, dataDir)
+    } catch (error: unknown) {
+      throw new HbdFileError((error as NodeJS.ErrnoException).message, this.#path, error)
+    }
+
+    this.#metadata = metadata
+    this.#dataDir = dataDir
+    this.#blocksDir = blocksDir
+
+    try {
+      this.#bat = BlockAllocationTable.fromBuffer(
+        await this.#handler.readFile(hashesPath),
+        this.getMaxBlockCount(),
+        options.force
+      )
+      this.#oldBlockCount = this.getMaxBlockCount()
+    } catch (error: unknown) {
+      this.#metadata = undefined
+      this.#dataDir = undefined
+      this.#blocksDir = undefined
+      throw new HbdFileError((error as NodeJS.ErrnoException).message, hashesPath, error)
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.#dirty) {
+      await this.flushMetadata()
+    }
+  }
+
+  async rename(newPath: string): Promise<void> {
+    newPath = normalize(newPath)
+    if (dirname(newPath) !== this.#diskDir) {
+      throw new Error(`can't move ${this.#path} out of its directory, to ${newPath}`)
+    }
+    await this.#handler.rename(this.#path, newPath)
+    this.#path = newPath
+  }
+
+  async readBlock(index: number): Promise<DiskBlock> {
+    const bat = this.#loadedBat
+    if (bat.isEmpty(index)) {
+      throw new Error(`no block at index ${index} of ${this.#path}`)
+    }
+    const hash = bat.hashAt(index)
+
+    // reads the whole file: that is exactly header + payload, and unlike a
+    // positional read it also works on an encrypted remote
+    const blockPath = this.#blockPath(hash)
+    const buffer = await this.#handler.readFile(blockPath)
+    let payload: Buffer
+    try {
+      ;({ payload } = decodeBlock(buffer, hash))
+    } catch (error: unknown) {
+      // says which file of which disk is broken
+      throw new HbdFileError(`${(error as Error).message} at index ${index} of ${this.#path}`, blockPath, error)
+    }
+
+    return { index, data: payload }
+  }
+
+  /**
+   * Purely additive: the previous hash at this index keeps its block file until
+   * flushMetadata notices no BAT entry references it any more.
+   */
+  async writeBlock({ index, data }: DiskBlock): Promise<number> {
+    const blockSize = this.getBlockSize()
+    if (data.length !== blockSize) {
+      throw new Error(`expected a ${blockSize} bytes block, got ${data.length}`)
+    }
+
+    const bat = this.#loadedBat
+    const hash = sha256hex(data)
+
+    // unchanged content at this index, nothing to do at all
+    if (!bat.isEmpty(index) && bat.hashAt(index) === hash) {
+      return blockSize
+    }
+
+    await this.#addBlockReference(hash, data)
+    this.#setHash(index, hash)
+
+    return blockSize
+  }
+
+  /**
+   * The child is usually a chain (see openDiskChain): only its public methods are used
+   */
+  async mergeBlock(childDisk: MergeChild, index: number, isResumingMerge: boolean): Promise<number> {
+    if (!childDisk.hasBlock(index)) {
+      return 0
+    }
+    const childHash = childDisk.getBlockHashAt(index)
+    if (this.hasBlock(index) && this.getBlockHashAt(index) === childHash) {
+      return 0
+    }
+
+    await this.#link(childDisk.getBlockPath(index), this.#blockPath(childHash))
+
+    this.#setHash(index, childHash)
+    return this.getBlockSize()
+  }
+
+  /**
+   * No-op: the BAT is built exclusively by writeBlock, which knows the hash.
+   * Declaring an index without content is meaningless in a content addressed
+   * format.
+   */
+  async setAllocatedBlocks(): Promise<void> {}
+
+  async resize(blockCount: number): Promise<void> {
+    if (blockCount === this.getMaxBlockCount()) {
+      return
+    }
+    const bat = this.#loadedBat
+    bat.resize(blockCount)
+    this.#metadata = { ...this.#loadedMetadata, virtualSize: blockCount * this.getBlockSize() }
+    this.#dirty = true
+  }
+
+  /**
+   * Writes the BAT to a new timestamped file, then points the hbd file at it.
+   * The previous hashes file is never overwritten, so a crash between the two
+   * writes leaves the disk readable through the old one. It is removed once
+   * the hbd points at the new one.
+   */
+  async flushMetadata(childDisk?: unknown): Promise<void> {
+    if (childDisk !== undefined) {
+      assertMergeChild(childDisk)
+      await this.#mergeWithChild(childDisk)
+    }
+    const metadata = this.#loadedMetadata
+    const dataDir = dirname(metadata.hashesPath)
+
+    // Every flush writes a new file: the one the hbd still points at must stay
+    // intact, so a crash before the hbd is updated leaves the disk readable.
+    // The name has a random part, a collision is only retried for safety.
+    const hashesPath: string = await pRetry(
+      async () => {
+        const path = join(dataDir, hashesFileName(new Date()))
+        await this.#handler.outputFile(this.#resolve(path), this.#loadedBat.toBuffer(), { flags: 'wx' })
+        return path
+      },
+      { delay: 0, tries: 3, when: { code: 'EEXIST' } }
+    )
+
+    const newMetadata = { ...metadata, hashesPath }
+    await this.#handler.outputStream(this.#path, Readable.from(JSON.stringify(newMetadata)), { checksum: false })
+    this.#metadata = newMetadata
+
+    await this.#removeOrphans(this.#resolve(metadata.hashesPath))
+    this.#oldBlockCount = this.getMaxBlockCount()
+
+    this.#dirty = false
+
+    // the hbd no longer points at it. Best effort: the data dir is claimed as
+    // a whole, so a file left by a failure or a crash is only removed by check()
+    try {
+      await this.#handler.unlink(this.#resolve(metadata.hashesPath), { checksum: false })
+    } catch (error) {
+      warn('failed to remove the previous hashes file', { path: this.#path, hashesPath: metadata.hashesPath, error })
+    }
+  }
+
+  async #mergeWithChild(childDisk: MergeChild): Promise<void> {
+    if (childDisk.getBlockSize() !== this.getBlockSize()) {
+      throw new Error(`can't merge a ${childDisk.getBlockSize()} bytes block disk into a ${this.getBlockSize()} one`)
+    }
+    // mergeBlock skips the indexes already holding the child's content
+    for (const index of childDisk.getBlockIndexes()) {
+      await this.mergeBlock(childDisk, index, true)
+    }
+  }
+
+  async clean({
+    remove = false,
+    logWarn = (message: string, data?: object) => {},
+    logInfo = (message: string, data?: object) => {},
+  } = {}): Promise<string | undefined> {
+    const current = basename(this.#loadedMetadata.hashesPath)
+    const leftovers: string[] = []
+    for (const filename of await this.#handler.list(this.#loadedDataDir)) {
+      if (filename !== current && /^hashes\..+\.hash$/.test(filename)) {
+        leftovers.push(join(this.#loadedDataDir, filename))
+      }
+    }
+    if (this.#loadedMetadata.dedupType === 'PER_BACKUP_REPOSITORY') {
+      const tmpDir = this.#tmpDir
+      for (const filename of await this.#handler.list(tmpDir, { ignoreMissing: true })) {
+        leftovers.push(join(this.#tmpDir, filename))
+      }
+    }
+    if (leftovers.length > 0) {
+      logInfo('leftover files of a hashed disk', {
+        disk: this.#path,
+        count: leftovers.length,
+        sample: leftovers.slice(0, LEFTOVER_SAMPLE),
+      })
+    }
+    if (remove) {
+      for (const path of leftovers) {
+        debug('leftover file of a hashed disk', { disk: this.#path, path })
+        await this.#handler
+          .unlink(path, { checksum: false })
+          .catch(error => logWarn('failed to remove a leftover file', { path, error }))
+      }
+    }
+    return undefined
+  }
+
+  async mergeMetadata(childDisk: unknown): Promise<void> {
+    assertMergeChild(childDisk)
+    // on a chain, the metadata of its newest disk
+    const { uuid, virtualSize } = childDisk.getMetadata()
+    if (Math.ceil(virtualSize / this.getBlockSize()) !== this.getMaxBlockCount()) {
+      throw new Error(`the BAT of ${this.#path} is not sized for its child, resize first`)
+    }
+    const metadata = { ...this.#loadedMetadata, uuid, virtualSize }
+    await this.#handler.outputStream(this.#path, Readable.from(JSON.stringify(metadata)), { checksum: false })
+    this.#metadata = metadata
+  }
+
+  /**
+   * What this disk claims inside `dir`: the hbd file, and its data directory as
+   * a whole. Used by lineage and remote cleanup to tell owned files from orphans.
+   */
+  async listAssociatedFiles(dir: string): Promise<Array<string>> {
+    const files = [this.#path, this.#loadedDataDir]
+
+    return files.filter(p => isInDir(p, dir))
+  }
+
+  async unlink(): Promise<void> {
+    if (this.#loadedMetadata.dedupType === 'PER_BACKUP_REPOSITORY') {
+      for (const index of this.#loadedBat.indexes()) {
+        try {
+          await this.#removeBlockReference(this.#loadedBat.hashAt(index))
+        } catch (error) {
+          warn('failed to release block', { path: this.#path, index, error })
+        }
+      }
+    }
+    await this.#handler.unlink(this.#path)
+    await this.#handler.rmtree(this.#loadedDataDir)
+
+    this.#metadata = undefined
+    this.#bat = undefined
+    this.#dataDir = undefined
+    this.#blocksDir = undefined
+    this.#dirty = false
+  }
+
+  // ---------------------------------------------------------------- private
+
+  /**
+   * Resolves a path stored in the metadata, which is relative to the hbd file.
+   */
+  #resolve(relativePath: string, container: string = this.#diskDir): string {
+    const resolved = normalize(join(this.#diskDir, relativePath))
+    if (!isInDir(resolved, container)) {
+      throw new Error(`path ${relativePath} escapes ${container}`)
+    }
+    return resolved
+  }
+
+  #blockPath(hash: BlockHash): string {
+    if (this.#blocksDir === undefined) {
+      throw new Error(`can't use a HashedDiskDeduplicated before init`)
+    }
+    return join(this.#blocksDir, blockRelPath(hash))
+  }
+
+  get #tmpDir(): string {
+    if (this.#blocksDir === undefined) {
+      throw new Error(`can't use a HashedDiskDeduplicated before init`)
+    }
+    return join(this.#blocksDir, '.tmp')
+  }
+
+  /**
+   * @param hash always hex
+   * @returns complete path in store
+   * Rooted at the remote root, not the disk dir, so no #resolve: the root comes from
+   * the caller and the hash is always hex
+   */
+  #storePath(hash: BlockHash, copy = 0): string {
+    return join(this.#blockStorePath, blockRelPath(hash), String(copy))
+  }
+
+  async #link(existingPath: string, newPath: string): Promise<void> {
+    try {
+      await this.#handler.link(existingPath, newPath)
+    } catch (error: unknown) {
+      // EEXIST => block already referenced by disk
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST') {
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Writes the block file unless it is already there. 'wx' for concurrent cases
+   */
+  async #storeBlock(hash: BlockHash, data: Buffer): Promise<void> {
+    try {
+      await this.#handler.outputFile(this.#blockPath(hash), encodeBlock(hash, data), {
+        flags: 'wx',
+      })
+    } catch (error: unknown) {
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST') {
+        throw error
+      }
+      // already stored, by another index of this disk or by a previous run
+    }
+  }
+
+  async #addBlockReference(hash: BlockHash, data: Buffer): Promise<void> {
+    if (this.#loadedMetadata.dedupType === 'PER_DISK') {
+      return this.#storeBlock(hash, data)
+    }
+
+    const blockPath = this.#blockPath(hash)
+    for (let copy = 0; ; copy++) {
+      const storePath = this.#storePath(hash, copy)
+      try {
+        await this.#link(storePath, blockPath)
+      } catch (error: unknown) {
+        const { code } = error as NodeJS.ErrnoException
+        if (code === 'EMLINK') {
+          continue
+        }
+        if (code !== 'ENOENT') {
+          throw error
+        }
+
+        // link did not work, we need a new block
+        const tmp = join(this.#tmpDir, randomUUID())
+        await this.#handler.outputFile(tmp, encodeBlock(hash, data), { flags: 'wx' })
+        try {
+          await this.#link(tmp, storePath)
+          await this.#link(storePath, blockPath)
+        } finally {
+          await this.#handler
+            .unlink(tmp, { checksum: false })
+            .catch(error => warn('failed to remove a temporary block file', { path: this.#path, tmp, error }))
+        }
+        return
+      }
+
+      if (this.#scrubEnabled) {
+        await this.#scrub(hash, data, storePath)
+      }
+      return
+    }
+  }
+
+  /**
+   * best effort: the new BAT is already safe, a failure only leaks
+   */
+  async #removeOrphans(previousHashesPath: string): Promise<void> {
+    const stream = await this.#handler.createReadStream(previousHashesPath)
+    try {
+      const current = this.#loadedBat
+      const candidates = new HashEntrySet()
+
+      // Pass 1: Entries of the previous BAT this flush replaced
+      for await (const { firstIndex, bat } of BlockAllocationTable.readChunks(stream, this.#oldBlockCount)) {
+        for (const index of bat.indexes()) {
+          if (!current.sameAt(firstIndex + index, bat, index)) {
+            candidates.add(Buffer.from(bat.entryAt(index)))
+          }
+        }
+      }
+
+      // Pass 2: A replaced hash can still be referenced at another index
+      for (const index of current.indexes()) {
+        if (candidates.size === 0) {
+          break
+        }
+        candidates.delete(current.entryAt(index))
+      }
+
+      // Removal
+      for (const entry of candidates) {
+        await this.#removeBlockReference(entry.toString('hex') as BlockHash)
+      }
+    } catch (error) {
+      warn('failed to remove orphaned blocks', { path: this.#path, error })
+    } finally {
+      stream.destroy()
+    }
+  }
+
+  /**
+   * @param options.skipStoreCheck the caller knows another disk still links
+   * this block (a parent holding the same hash during a merge), so the store
+   * file cannot be the last copy: saves one stat per block
+   */
+  async #removeBlockReference(hash: BlockHash, { skipStoreCheck = false } = {}): Promise<void> {
+    const blockPath = this.#blockPath(hash)
+    if (this.#loadedMetadata.dedupType === 'PER_DISK' || skipStoreCheck) {
+      return this.#handler.unlink(blockPath, { checksum: false })
+    } else {
+      let nlink: number
+      nlink = await this.#handler.getLinkCount(blockPath)
+      if (nlink === 0) {
+        return // already removed
+      }
+      await this.#handler.unlink(blockPath, { checksum: false })
+      if (nlink === 2) {
+        await this.#removeOrphanStoreCopy(hash)
+      }
+    }
+  }
+
+  async #removeOrphanStoreCopy(hash: BlockHash): Promise<void> {
+    const first = this.#storePath(hash)
+    const dir = dirname(first)
+    // first, check /0
+    if ((await this.#linkCount(first)) === 1) {
+      await this.#handler.unlink(first, { checksum: false })
+    } else {
+      // if /0 copy is not used for this disk, check the other paths
+      for (const name of await this.#handler.list(dir, { ignoreMissing: true })) {
+        if (name === '0') {
+          continue
+        }
+        const path = join(dir, name)
+        if ((await this.#linkCount(path)) === 1) {
+          await this.#handler.unlink(path, { checksum: false })
+        }
+      }
+    }
+    await this.#handler.rmdir(dir).catch(error => {
+      if (error.code !== 'ENOTEMPTY' && error.code !== 'EEXIST') {
+        warn('failed to remove an empty store directory', { dir, error })
+      }
+    })
+  }
+
+  async #linkCount(path: string): Promise<number> {
+    return this.#handler.getLinkCount(path).catch(error => {
+      warn('failed to read a store copy link count', { path, error })
+      return 0
+    })
+  }
+
+  #setHash(index: number, hash: BlockHash): void {
+    this.#loadedBat.set(index, hash)
+    this.#dirty = true
+  }
+
+  async #scrub(hash: BlockHash, data: Buffer, storePath: string): Promise<void> {
+    try {
+      const headerBuffer = Buffer.alloc(HBD_HEADER_SIZE)
+      const now = Date.now()
+      // read metadata
+      await this.#handler.read(storePath, headerBuffer, 0)
+      const header = parseBlockHeader(headerBuffer)
+
+      // a header we can't trust (wrong magic, another hash, a date in the future)
+      // must not let its verifiedAt skip the check
+      const trusted = header.magic === HBD_MAGIC && header.payloadHash === hash && header.verifiedAt <= now
+      if (trusted && now - header.verifiedAt < this.#scrubMaxAgeMs) {
+        return
+      }
+
+      this.#blocksScrubbed++
+
+      // check block
+      const buffer = await this.#handler.readFile(storePath)
+
+      let corrupted = false
+      try {
+        decodeBlock(buffer, hash)
+      } catch {
+        corrupted = true
+      }
+
+      // update medata or rewrite block
+      if (!corrupted) {
+        await this.#handler.write(storePath, setVerifiedAt(headerBuffer, now), 0)
+      } else {
+        this.#blocksCorrupted++
+        warn('corrupted store block, repaired', { hash, storePath })
+        await this.#handler.outputFile(storePath, encodeBlock(hash, data, CODEC_RAW, now), {
+          flags: 'w',
+        })
+      }
+    } catch (error) {
+      warn('failed to scrub a store block', { hash, storePath, error })
+    }
+  }
+}

@xen-orchestra/backup-archive/src/disks/hashed/hbdPaths.mts

@@ -0,0 +1,231 @@
+// Offset  Size  Field
+//      0     4  magic: 0x48424400 ("HBD\0")
+//      4     1  codec: 0x00 = raw (no compression),0x01= brotli compression  ,  reserved for future codecs,
+//      5     3  reserved for compression flags (zero)
+//      8     8  verified_at: Unix timestamp in milliseconds of last successful hash check (0 = never)
+//     16    32  payload_hash: SHA-256 of the raw (uncompressed, unencrypted) payload — redundant with
+//               the store path but enables self-contained verification without path context
+//     48     4  payload_length: bytes stored after the header, uint32 LE (never 0)
+//     52   460  reserved (zero)
+//    512     N  payload: raw block data (currently uncompressed, unencrypted; N = blockSize)
+import crypto from 'node:crypto'
+import { validate as validateUuid } from 'uuid'
+import type { Branded } from '@vates/types'
+import { compareVersions } from 'compare-versions'
+
+export type BlockHash = Branded<'hbd-block-hash'>
+
+export const VERSION = '0.0.1'
+export const HBD_HEADER_SIZE = 512
+export const HBD_MAGIC = 0x48424400
+export const HASH_SIZE = 32 // bytes, on disk
+// the block store, at the remote root: shared by every hashed disk of the remote
+export const BLOCK_STORE_PATH = 'xo-block-store'
+
+// block header field offsets, in bytes (see the layout table at the top of this file)
+const OFFSET_MAGIC = 0
+const OFFSET_CODEC = 4
+const OFFSET_VERIFIED_AT = 8
+const OFFSET_PAYLOAD_HASH = 16
+const OFFSET_PAYLOAD_LENGTH = 48
+
+export const CODEC_RAW = 0x00
+export const CODEC_BROTLI = 0x01
+
+const HASH_HEX_LENGTH = HASH_SIZE * 2 // chars, in memory
+const BLOCK_HASH_RE = new RegExp(`^[0-9a-f]{${HASH_HEX_LENGTH}}$`)
+
+export function asBlockHash(value: string): BlockHash {
+  if (!BLOCK_HASH_RE.test(value)) {
+    throw new Error(`not a valid block hash: ${value}`)
+  }
+  return value as BlockHash
+}
+
+export type DedupType = 'PER_BACKUP_REPOSITORY' | 'PER_DISK'
+
+export interface HashedDiskMetadata {
+  version: string
+  virtualSize: number
+  blockSize: number
+  uuid: string
+  parentUuid?: string
+  parentPath?: string // relative path to parent hbd file from this hbd file
+  dedupType: DedupType
+  localBlocksPath: string //relative to this file
+  hashesPath: string //relative to this file
+}
+
+export class HbdFileError extends Error {
+  path: string
+
+  constructor(message: string, path: string, cause?: unknown) {
+    super(`${message} (in ${path})`, { cause })
+    this.path = path
+  }
+}
+
+export function checkVersion(version: string) {
+  if (compareVersions(version, VERSION) > 0) {
+    throw new Error(`Unsupported hbd version ${version}, expected: ${VERSION}`)
+  }
+}
+
+/**
+ * Directory holding everything a disk owns (hashes files and blocks), relative
+ * to its hbd file. Derived from the uuid rather than read from the metadata, so
+ * it is the same directory whatever the metadata claims.
+ */
+export function dataDirName(uuid: string): string {
+  if (!validateUuid(uuid)) {
+    throw new Error(`not a valid uuid: ${uuid}`)
+  }
+  return `data/${uuid}`
+}
+
+export function blockRelPath(hash: BlockHash): string {
+  return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`
+}
+
+export interface HbdBlockHeader {
+  magic: number
+  codec: number
+  verifiedAt: number // ms since epoch, 0 = never verified
+  payloadHash: BlockHash
+  payloadLength: number // bytes stored after the header
+}
+
+export function buildBlockHeader(hash: BlockHash, payloadLength: number, codec = CODEC_RAW, verifiedAt = 0): Buffer {
+  if (!Number.isInteger(codec) || codec < 0 || codec > 0xff) {
+    throw new Error(`codec must be a single byte, got ${codec}`)
+  }
+  if (!Number.isSafeInteger(verifiedAt) || verifiedAt < 0) {
+    throw new Error(`verifiedAt must be a non negative safe integer, got ${verifiedAt}`)
+  }
+  if (!Number.isInteger(payloadLength) || payloadLength <= 0 || payloadLength > 0xffffffff) {
+    throw new Error(`payloadLength must be a positive 32 bits integer, got ${payloadLength}`)
+  }
+
+  // zero filled, which is exactly what the reserved ranges (5..7 and 52..511) must contain
+  const header = Buffer.alloc(HBD_HEADER_SIZE)
+
+  // big endian so the bytes read as "HBD\0" in a hex dump
+  header.writeUInt32BE(HBD_MAGIC, OFFSET_MAGIC)
+  header.writeUInt8(codec, OFFSET_CODEC)
+  header.writeBigInt64LE(BigInt(verifiedAt), OFFSET_VERIFIED_AT)
+  header.writeUInt32LE(payloadLength, OFFSET_PAYLOAD_LENGTH)
+
+  if (header.write(hash, OFFSET_PAYLOAD_HASH, 'hex') !== HASH_SIZE) {
+    throw new Error(`not a valid block hash: ${hash}`)
+  }
+
+  return header
+}
+
+/**
+ * Accepts a buffer of at least HBD_HEADER_SIZE bytes, so a full block
+ * (header + payload) can be passed without slicing it first.
+ *
+ * Does not validate `magic`: it is returned so the caller decides what a
+ * mismatch means. Any caller reading a payload MUST check it.
+ */
+export function parseBlockHeader(buffer: Buffer): HbdBlockHeader {
+  if (buffer.length < HBD_HEADER_SIZE) {
+    throw new Error(`a block header is ${HBD_HEADER_SIZE} bytes, got ${buffer.length}`)
+  }
+
+  return {
+    magic: buffer.readUInt32BE(OFFSET_MAGIC),
+    codec: buffer.readUInt8(OFFSET_CODEC),
+    verifiedAt: Number(buffer.readBigInt64LE(OFFSET_VERIFIED_AT)),
+    payloadHash: buffer.subarray(OFFSET_PAYLOAD_HASH, OFFSET_PAYLOAD_HASH + HASH_SIZE).toString('hex') as BlockHash,
+    payloadLength: buffer.readUInt32LE(OFFSET_PAYLOAD_LENGTH),
+  }
+}
+
+/** a whole block file: its header, with the payload length recorded, then the payload */
+export function encodeBlock(hash: BlockHash, data: Buffer, codec = CODEC_RAW, verifiedAt = 0): Buffer {
+  return Buffer.concat([buildBlockHeader(hash, data.length, codec, verifiedAt), data])
+}
+
+/**
+ * Validates a full block file (header + payload) and returns its payload.
+ *
+ * `expectedHash` is the hash the BAT holds for this index: it is the authority,
+ * the header is only cross checked against it. Pure, no I/O — the caller reads
+ * the bytes and decides what to do with a throw.
+ */
+export function decodeBlock(buffer: Buffer, expectedHash: BlockHash): { header: HbdBlockHeader; payload: Buffer } {
+  if (buffer.length < HBD_HEADER_SIZE) {
+    throw new Error(`truncated block: expected at least ${HBD_HEADER_SIZE} bytes, got ${buffer.length}`)
+  }
+
+  const header = parseBlockHeader(buffer)
+  if (header.magic !== HBD_MAGIC) {
+    throw new Error(`not a hbd block: magic is 0x${header.magic.toString(16)}`)
+  }
+  // the file is checked against its own header only
+  // Every block records its length, 0 means a damaged header
+  const { payloadLength } = header
+  if (payloadLength === 0) {
+    throw new Error('not a hbd block: no payload length')
+  }
+  const expectedLength = HBD_HEADER_SIZE + payloadLength
+  if (buffer.length < expectedLength) {
+    throw new Error(`truncated block: expected ${expectedLength} bytes, got ${buffer.length}`)
+  }
+  if (header.payloadHash !== expectedHash) {
+    // wrong file at the right path, or a store file overwritten by another hash
+    throw new Error(`block hash mismatch: header says ${header.payloadHash}, expected ${expectedHash}`)
+  }
+
+  const payload = buffer.subarray(HBD_HEADER_SIZE, expectedLength)
+
+  // Only raw payloads can be checked here. An authenticated encryption codec
+  // gets the same guarantee from its auth tag at decryption time.
+  if (header.codec === CODEC_RAW) {
+    const actualHash = sha256hex(payload)
+    if (actualHash !== expectedHash) {
+      throw new Error(`block corruption on read: expected ${expectedHash}, got ${actualHash}`)
+    }
+  }
+
+  return { header, payload }
+}
+
+export function hashesFileName(date: Date): string {
+  return `hashes.${date.getTime()}.${crypto.randomBytes(4).toString('hex')}.hash`
+}
+
+export function sha256hex(data: Buffer): BlockHash {
+  const hash = crypto.createHash('sha256')
+  return hash.update(data).digest('hex') as BlockHash
+}
+
+export function setVerifiedAt(header: Buffer, verifiedAt: number): Buffer {
+  header.writeBigInt64LE(BigInt(verifiedAt), OFFSET_VERIFIED_AT)
+  return header
+}
+
+export class HashEntrySet {
+  #keys = new Set<string>()
+
+  add(entry: Buffer): this {
+    this.#keys.add(entry.toString('latin1'))
+    return this
+  }
+  has(entry: Buffer): boolean {
+    return this.#keys.has(entry.toString('latin1'))
+  }
+  delete(entry: Buffer): boolean {
+    return this.#keys.delete(entry.toString('latin1'))
+  }
+  get size(): number {
+    return this.#keys.size
+  }
+  *[Symbol.iterator](): Iterator<Buffer> {
+    for (const key of this.#keys) {
+      yield Buffer.from(key, 'latin1')
+    }
+  }
+}

@xen-orchestra/backup-archive/src/disks/hashed/index.mts

@@ -0,0 +1,7 @@
+export { BlockAllocationTable } from './BlockAllocationTable.mjs'
+export { HashedDisk } from './HashedDisk.mjs'
+export { HashedDiskChain } from './HashedDiskChain.mjs'
+export { HashedDiskDeduplicated } from './HashedDiskDeduplicated.mjs'
+export * from './hbdPaths.mjs'
+export { scanBlockStore } from './scanBlockStore.mjs'
+export { writeHbd } from './writeHbd.mjs'

@xen-orchestra/backup-archive/src/disks/hashed/scanBlockStore.mts

@@ -0,0 +1,86 @@
+import { asyncEach } from '@vates/async-each'
+import type { RemoteHandlerAbstract } from '@xen-orchestra/fs'
+import { basename, join, normalize } from '@xen-orchestra/fs/path'
+import { asBlockHash, BLOCK_STORE_PATH, decodeBlock } from './hbdPaths.mjs'
+
+// <store>/<ab>/<cd>/<hash>/<copy>: copies are always at this depth
+const COPY_DEPTH = 4
+
+export async function scanBlockStore(
+  handler: RemoteHandlerAbstract,
+  {
+    remove = false,
+    verify = false,
+    logInfo = () => {},
+    logWarn = () => {},
+    logDebug = () => {},
+  }: {
+    remove?: boolean
+    verify?: boolean
+    logInfo?: (message: string, data?: object) => void
+    logWarn?: (message: string, data?: object) => void
+    logDebug?: (message: string, data?: object) => void
+  } = {}
+): Promise<{ removed: number }> {
+  let removed = 0
+
+  async function visit(dir: string, depth: number): Promise<void> {
+    let names: string[]
+    try {
+      names = await handler.list(dir, { ignoreMissing: true })
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOTDIR') {
+        return // not a store directory, left alone
+      }
+      throw error
+    }
+    await asyncEach(
+      names,
+      async name => {
+        const path = join(dir, name)
+        if (depth + 1 < COPY_DEPTH) {
+          await visit(path, depth + 1) // a hash level: go down
+        } else if (depth + 1 === COPY_DEPTH) {
+          if (await removeIfOrphan(path)) {
+            removed++ // a store copy
+          } else if (verify) {
+            // <ab>/<cd>/<hash>/<copy>: the copy's directory is named after the full hash
+            await verifyCopy(path, basename(dir))
+          }
+        }
+      },
+      {
+        concurrency: 2,
+      }
+    )
+  }
+
+  async function removeIfOrphan(path: string): Promise<boolean> {
+    try {
+      if ((await handler.getLinkCount(path)) !== 1) {
+        return false // still linked by a disk
+      }
+      logDebug('orphan store file', { path })
+      if (remove) {
+        await handler.unlink(path, { checksum: false })
+      }
+      return true
+    } catch (error) {
+      logDebug('failed to check a store copy', { path, error })
+      return false
+    }
+  }
+
+  async function verifyCopy(path: string, hashCopy: string): Promise<void> {
+    try {
+      const hash = asBlockHash(hashCopy)
+      const buffer = await handler.readFile(path)
+      decodeBlock(buffer, hash)
+    } catch (error) {
+      logWarn('corrupted store block', { path, error })
+    }
+  }
+
+  await visit(normalize(BLOCK_STORE_PATH), 0)
+  return { removed }
+}

@xen-orchestra/backup-archive/src/disks/hashed/writeHbd.mts

@@ -0,0 +1,93 @@
+import type { Disk, DiskBlock } from '@xen-orchestra/disk-transform'
+import { type RemoteHandlerAbstract } from '@xen-orchestra/fs'
+import { asyncEach } from '@vates/async-each'
+import { createLogger } from '@xen-orchestra/log'
+
+import { HashedDiskDeduplicated } from './HashedDiskDeduplicated.mjs'
+
+const { warn } = createLogger('xo:backup-archive:hbd')
+
+/**
+ * Writes every block of `source` to a new hbd disk at `path`, sharing its
+ * blocks with the whole backup repository.
+ *
+ * On failure, the partial disk is removed and the blocks it already
+ * referenced are released.
+ *
+ * @returns the number of bytes read from the source
+ */
+export async function writeHbd(
+  handler: RemoteHandlerAbstract,
+  path: string,
+  source: Disk,
+  {
+    uuid,
+    parentUuid,
+    parentPath,
+    concurrency = 16,
+    validator = async () => {},
+    signal,
+  }: {
+    uuid: string
+    parentUuid?: string
+    parentPath?: string
+    concurrency?: number
+    validator?: (path: string) => Promise<void>
+    signal?: AbortSignal
+  }
+): Promise<number> {
+  let disk: HashedDiskDeduplicated | undefined
+  // asyncEach rejects on the first error without waiting for the other
+  // iterations: the cleanup must not run while a block is still being
+  // referenced, or that reference would never be released
+  const pendingWrites = new Set<Promise<number>>()
+  try {
+    signal?.throwIfAborted()
+    const blockSize = source.getBlockSize()
+    disk = await HashedDiskDeduplicated.create({
+      handler,
+      path,
+      virtualSize: source.getVirtualSize(),
+      blockSize,
+      uuid,
+      parentUuid,
+      parentPath,
+      dedupType: 'PER_BACKUP_REPOSITORY',
+    })
+
+    let transferred = 0
+    const target = disk
+    await asyncEach(
+      source.diskBlocks(),
+      async ({ index, data }: DiskBlock) => {
+        signal?.throwIfAborted()
+        transferred += data.length
+        // the last block of a disk whose size is not a multiple of the block size
+        if (data.length < blockSize) {
+          const padded = Buffer.alloc(blockSize)
+          data.copy(padded)
+          data = padded
+        }
+        const write = target.writeBlock({ index, data })
+        pendingWrites.add(write)
+        try {
+          await write
+        } finally {
+          pendingWrites.delete(write)
+        }
+      },
+      { concurrency }
+    )
+
+    await disk.flushMetadata()
+    await validator(path)
+    return transferred
+  } catch (error) {
+    await source.close().catch(() => {})
+    await Promise.allSettled(pendingWrites)
+    if (disk !== undefined) {
+      await disk.unlink().catch(unlinkError => warn('failed to remove partial hbd', { path, error: unlinkError }))
+    }
+    throw error
+  }
+}

@xen-orchestra/backup-archive/src/disks/hashed/writeHbd.test.mjs

@@ -0,0 +1,198 @@
+import { test } from 'node:test'
+import { strict as assert } from 'node:assert'
+
+import tmp from 'tmp'
+import * as uuid from 'uuid'
+import { getHandler } from '@xen-orchestra/fs'
+import { normalize } from '@xen-orchestra/fs/path'
+import { pFromCallback } from 'promise-toolbox'
+import { rimraf } from 'rimraf'
+
+import { BLOCK_STORE_PATH, HashedDiskDeduplicated, writeHbd } from '@xen-orchestra/backup-archive/disks/hashed'
+
+const { beforeEach, afterEach, describe } = test
+
+const BLOCK_SIZE = 4096
+
+let tempDir, handler, diskDir, diskPath
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  diskDir = `xo-vm-backups/VMUUID/vdis/${uuid.v4()}`
+  diskPath = `${diskDir}/20260814T120000000Z.hbd`
+})
+
+afterEach(async () => {
+  await handler.forget()
+  await rimraf(tempDir)
+})
+
+/**
+ * Minimal source disk: yields `blocks` in order, then optionally throws
+ */
+function fakeSource(blocks, { virtualSize = blocks.reduce((sum, b) => sum + b.length, 0), failAfter } = {}) {
+  let closed = false
+  return {
+    get closed() {
+      return closed
+    },
+    getBlockSize: () => BLOCK_SIZE,
+    getVirtualSize: () => virtualSize,
+    async *diskBlocks() {
+      for (const [index, data] of blocks.entries()) {
+        if (index === failAfter) {
+          throw new Error('source failure')
+        }
+        yield { index, data }
+      }
+    },
+    async close() {
+      closed = true
+    },
+  }
+}
+
+// what is left of the disk dir: `data/` itself is shared with the other disks of the dir
+async function listDiskDir() {
+  const list = async path => (await handler.list(path).catch(() => [])).sort()
+  return { files: await list(diskDir), data: await list(`${diskDir}/data`) }
+}
+
+async function listStore() {
+  const entries = []
+  for (const ab of await handler.list(BLOCK_STORE_PATH).catch(() => [])) {
+    for (const cd of await handler.list(`${BLOCK_STORE_PATH}/${ab}`)) {
+      entries.push(...(await handler.list(`${BLOCK_STORE_PATH}/${ab}/${cd}`)))
+    }
+  }
+  return entries
+}
+
+describe('writeHbd', () => {
+  test('writes every block and reads them back', async () => {
+    const blocks = [Buffer.alloc(BLOCK_SIZE, 1), Buffer.alloc(BLOCK_SIZE, 2), Buffer.alloc(BLOCK_SIZE, 3)]
+    const transferred = await writeHbd(handler, diskPath, fakeSource(blocks), { uuid: uuid.v4() })
+
+    assert.equal(transferred, 3 * BLOCK_SIZE)
+
+    const disk = new HashedDiskDeduplicated({ handler, path: diskPath })
+    await disk.init()
+    assert.deepEqual(disk.getBlockIndexes(), [0, 1, 2])
+    assert.equal(disk.getVirtualSize(), 3 * BLOCK_SIZE)
+    assert.equal(disk.getMetadata().dedupType, 'PER_BACKUP_REPOSITORY')
+    for (const [index, data] of blocks.entries()) {
+      assert.deepEqual((await disk.readBlock(index)).data, data)
+    }
+  })
+
+  test('pads a short last block with zeros', async () => {
+    const blocks = [Buffer.alloc(BLOCK_SIZE, 1), Buffer.alloc(BLOCK_SIZE / 2, 2)]
+    const transferred = await writeHbd(handler, diskPath, fakeSource(blocks), { uuid: uuid.v4() })
+
+    // counts what was read from the source, not the padding
+    assert.equal(transferred, BLOCK_SIZE + BLOCK_SIZE / 2)
+
+    const disk = new HashedDiskDeduplicated({ handler, path: diskPath })
+    await disk.init()
+    const { data } = await disk.readBlock(1)
+    assert.equal(data.length, BLOCK_SIZE)
+    assert.deepEqual(data.subarray(0, BLOCK_SIZE / 2), Buffer.alloc(BLOCK_SIZE / 2, 2))
+    assert.deepEqual(data.subarray(BLOCK_SIZE / 2), Buffer.alloc(BLOCK_SIZE / 2, 0))
+  })
+
+  test('identical blocks share a single store entry', async () => {
+    const blocks = [Buffer.alloc(BLOCK_SIZE, 1), Buffer.alloc(BLOCK_SIZE, 1), Buffer.alloc(BLOCK_SIZE, 2)]
+    await writeHbd(handler, diskPath, fakeSource(blocks), { uuid: uuid.v4() })
+
+    assert.equal((await listStore()).length, 2)
+  })
+
+  test('stores parent uuid and path', async () => {
+    const parentUuid = uuid.v4()
+    await writeHbd(handler, diskPath, fakeSource([Buffer.alloc(BLOCK_SIZE, 1)]), {
+      uuid: uuid.v4(),
+      parentUuid,
+      parentPath: '20260813T120000000Z.hbd',
+    })
+
+    const disk = new HashedDiskDeduplicated({ handler, path: diskPath })
+    await disk.init()
+    assert.equal(disk.isDifferencing(), true)
+    assert.equal(disk.getParentUuid(), parentUuid)
+    assert.equal(disk.getParentPath(), normalize(`${diskDir}/20260813T120000000Z.hbd`))
+  })
+
+  test('calls the validator with the disk path once written', async () => {
+    let validatedPath
+    await writeHbd(handler, diskPath, fakeSource([Buffer.alloc(BLOCK_SIZE, 1)]), {
+      uuid: uuid.v4(),
+      validator: async path => {
+        // the disk must already be complete and readable
+        const disk = new HashedDiskDeduplicated({ handler, path })
+        await disk.init()
+        assert.deepEqual(disk.getBlockIndexes(), [0])
+        validatedPath = path
+      },
+    })
+
+    assert.equal(validatedPath, diskPath)
+  })
+
+  describe('on failure', () => {
+    test('removes the partial disk and releases its blocks', async () => {
+      const source = fakeSource([Buffer.alloc(BLOCK_SIZE, 7), Buffer.alloc(BLOCK_SIZE, 8)], {
+        virtualSize: 2 * BLOCK_SIZE,
+        failAfter: 1,
+      })
+      await assert.rejects(writeHbd(handler, diskPath, source, { uuid: uuid.v4() }), { message: 'source failure' })
+
+      assert.equal(source.closed, true)
+      assert.deepEqual(await listDiskDir(), { files: ['data'], data: [] })
+      assert.deepEqual(await listStore(), [])
+    })
+
+    test('keeps blocks still referenced by another disk', async () => {
+      const shared = Buffer.alloc(BLOCK_SIZE, 1)
+      const otherPath = `${diskDir}/20260813T120000000Z.hbd`
+      await writeHbd(handler, otherPath, fakeSource([shared]), { uuid: uuid.v4() })
+
+      const source = fakeSource([shared, Buffer.alloc(BLOCK_SIZE, 2)], { virtualSize: 2 * BLOCK_SIZE, failAfter: 1 })
+      await assert.rejects(writeHbd(handler, diskPath, source, { uuid: uuid.v4() }))
+
+      assert.equal((await listStore()).length, 1)
+      const other = new HashedDiskDeduplicated({ handler, path: otherPath })
+      await other.init()
+      assert.deepEqual((await other.readBlock(0)).data, shared)
+    })
+
+    test('removes the disk when the validator rejects', async () => {
+      await assert.rejects(
+        writeHbd(handler, diskPath, fakeSource([Buffer.alloc(BLOCK_SIZE, 1)]), {
+          uuid: uuid.v4(),
+          validator: async () => {
+            throw new Error('invalid')
+          },
+        }),
+        { message: 'invalid' }
+      )
+
+      assert.deepEqual(await listDiskDir(), { files: ['data'], data: [] })
+      assert.deepEqual(await listStore(), [])
+    })
+
+    test('does not start when the signal is already aborted', async () => {
+      const controller = new AbortController()
+      controller.abort()
+      await assert.rejects(
+        writeHbd(handler, diskPath, fakeSource([Buffer.alloc(BLOCK_SIZE, 1)]), {
+          uuid: uuid.v4(),
+          signal: controller.signal,
+        })
+      )
+
+      assert.deepEqual(await listDiskDir(), { files: [], data: [] })
+    })
+  })
+})

@xen-orchestra/backup-archive/src/disks/index.mjs

@@ -1,22 +1,16 @@
-import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
-
+import { instantiateDisk } from './instantiateDisk.mjs'
+export { isDisk } from './instantiateDisk.mjs'
 export { RemoteDisk } from './RemoteDisk.mjs'
 export { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
 export { openDiskChain } from './openDiskChain.mjs'
 export { MergeRemoteDisk } from './MergeRemoteDisk.mjs'
 
-const DISK_EXTENSIONS = ['.vhd']
-
 /**
  * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
  * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
  * @typedef {import('./MergeRemoteDisk.mjs').MergeState} MergeState
  */
 
-export function isDisk(path) {
-  return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
-}
-
 /**
  *
  * @param {Object} params
@@ -27,7 +21,7 @@ export function isDisk(path) {
  * @returns {Promise<Disposable<RemoteDisk>>}
  */
 export async function openDisposableDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
-  const disk = new RemoteVhdDisk({ handler, path })
+  const disk = instantiateDisk(handler, path)
   if (ignoreBlockIndexes) {
     // Best-effort init: return the disk even if init fails so that callers
     // can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
@@ -50,7 +44,7 @@ export async function openDisposableDisk({ handler, path, force = false, ignoreB
  * @returns {Promise<RemoteDisk>}
  */
 export async function openDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
-  const disk = new RemoteVhdDisk({ handler, path })
+  const disk = instantiateDisk(handler, path)
   if (ignoreBlockIndexes) {
     // Best-effort init: return the disk even if init fails so that callers
     // can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).

@xen-orchestra/backup-archive/src/disks/instantiateDisk.mjs

@@ -0,0 +1,24 @@
+// @ts-check
+/**
+ * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
+ * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
+ */
+import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
+// eslint rule off for src/**/*.mjs already
+import { HashedDiskDeduplicated } from './hashed/HashedDiskDeduplicated.mjs'
+
+const DISK_EXTENSIONS = ['.vhd', '.hbd']
+
+export function isDisk(path) {
+  return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
+}
+
+/**
+ * The disk format is chosen by the file extension, never by the remote config
+ * @param {RemoteHandlerAbstract} handler
+ * @param {string} path
+ * @returns {RemoteDisk}
+ */
+export function instantiateDisk(handler, path) {
+  return path.endsWith('.hbd') ? new HashedDiskDeduplicated({ handler, path }) : new RemoteVhdDisk({ handler, path })
+}

@xen-orchestra/backup-archive/src/disks/openDiskChain.mjs

@@ -4,9 +4,28 @@
  * @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
  * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
  */
-import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
 import { defer } from 'golike-defer'
+import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
 import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
+import { HashedDiskDeduplicated } from './hashed/HashedDiskDeduplicated.mjs'
+import { HashedDiskChain } from './hashed/HashedDiskChain.mjs'
+import { instantiateDisk } from './instantiateDisk.mjs'
+
+/**
+ * The chain class of the disks' format: a chain never mixes formats
+ * @param {string} path
+ * @param {RemoteDisk[]} disks
+ * @returns {RemoteDisk}
+ */
+function newChain(path, disks) {
+  if (disks.every(disk => disk instanceof HashedDiskDeduplicated)) {
+    return new HashedDiskChain({ disks })
+  }
+  if (disks.every(disk => disk instanceof RemoteVhdDisk)) {
+    return new RemoteVhdDiskChain({ disks })
+  }
+  throw new Error(`mixed disk formats in chain ${path}`)
+}
 
 /**
  * @param {any} $defer
@@ -20,15 +39,15 @@ import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
  */
 async function _openDiskChain($defer, { handler, path, until, force = false, ignoreBlockIndexes = false }) {
   /**
-   * @type {RemoteVhdDisk}
+   * @type {RemoteDisk}
    */
   let disk
   /**
-   * @type {Array<RemoteVhdDisk>}
+   * @type {Array<RemoteDisk>}
    */
   const disks = []
   $defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
-  disk = new RemoteVhdDisk({ handler, path })
+  disk = instantiateDisk(handler, path)
 
   await disk.init({ force, ignoreBlockIndexes })
   disks.push(disk)
@@ -40,15 +59,14 @@ async function _openDiskChain($defer, { handler, path, until, force = false, ign
     }
     // Instantiate + init the parent directly (instead of openParent()) so the
     // block allocation table read can be skipped when ignoreBlockIndexes is set.
-    disk = /** @type {RemoteVhdDisk} */ (disk.instantiateParent())
+    disk = disk.instantiateParent()
     await disk.init({ force, ignoreBlockIndexes })
     disks.unshift(disk)
   }
   if (!foundRootDisk) {
     throw new Error(`trying to open chain from ${path} to ${until}, root disk not found`)
   }
-  // the root disk
-  return new RemoteVhdDiskChain({ disks })
+  return newChain(path, disks)
 }
 
 /**

@xen-orchestra/backup-archive/tsconfig.json

@@ -14,6 +14,6 @@
     "types": ["node"] // Only use types from Node.js and packages that are explicitly imported
   },
   "include": ["src/**/*"],
-  // *.integ.mjs deliberately imports from the built dist/ output to test it
-  "exclude": ["node_modules", "dist", "**/*.integ.mjs"]
+  // *.test.mjs and *.integ.mjs deliberately import from the built dist/ output to test it
+  "exclude": ["node_modules", "dist", "**/*.test.mjs", "**/*.integ.mjs"]
 }

@xen-orchestra/backups/RemoteAdapter.integ.mjs

@@ -8,6 +8,8 @@ import { getHandler } from '@xen-orchestra/fs'
 import { relativeFromFile } from '@xen-orchestra/fs/path'
 import { Disposable, pFromCallback } from 'promise-toolbox'
 import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
+import { openDisposableDisk } from '@xen-orchestra/backup-archive/disks'
+import { BLOCK_STORE_PATH, HashedDiskDeduplicated } from '@xen-orchestra/backup-archive/disks/hashed'
 import { RemoteAdapter } from './RemoteAdapter.mjs'
 import { checkDisk } from './_runners/_writers/_checkDisk.mjs'
 import { formatFilenameDate } from './_filenameDate.mjs'
@@ -40,8 +42,18 @@ afterEach(async () => {
 const uniqueId = () => uuid.v1()
 const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
 
-async function getAdapter({ useVhdDirectory = false, vhdDirectoryCompression, remoteCompressionType } = {}) {
-  handler = getHandler({ url: `file://${tempDir}`, useVhdDirectory, compressionType: remoteCompressionType })
+async function getAdapter({
+  useVhdDirectory = false,
+  diskFormat,
+  vhdDirectoryCompression,
+  remoteCompressionType,
+} = {}) {
+  handler = getHandler({
+    url: `file://${tempDir}`,
+    useVhdDirectory,
+    diskFormat,
+    compressionType: remoteCompressionType,
+  })
   await handler.sync()
   return new RemoteAdapter(handler, { vhdDirectoryCompression })
 }
@@ -163,6 +175,56 @@ describe('RemoteAdapter#isMergeableParent', { concurrency: 1 }, () => {
   }
 })
 
+// a folder can hold disks of several formats, but a chain never mixes them: only a disk
+// of the format this remote writes can be the parent of the next delta
+describe('RemoteAdapter#isMergeableParent across disk formats', { concurrency: 1 }, () => {
+  const COMPRESSION = 'brotli'
+  // disk kind on the remote -> the only remote format that may chain onto it
+  const DISK_KINDS = {
+    file: 'vhd',
+    directory: 'vhdDirectory',
+    hbd: 'hbd',
+  }
+
+  async function createParent(kind, diskUuid) {
+    if (kind === 'hbd') {
+      const path = `${basePath}/disk.hbd`
+      const disk = await HashedDiskDeduplicated.create({
+        handler,
+        path,
+        virtualSize: DEFAULT_BLOCK_SIZE,
+        blockSize: DEFAULT_BLOCK_SIZE,
+        uuid: uuid.stringify(diskUuid),
+        dedupType: 'PER_BACKUP_REPOSITORY',
+      })
+      await disk.close()
+      return path
+    }
+    await generateVhd(`${basePath}/disk.vhd`, {
+      mode: kind,
+      compression: kind === 'directory' ? COMPRESSION : undefined,
+      uuid: diskUuid,
+    })
+    return kind === 'directory' ? `${basePath}/disk.vhd.alias.vhd` : `${basePath}/disk.vhd`
+  }
+
+  for (const [kind, matchingFormat] of Object.entries(DISK_KINDS)) {
+    for (const diskFormat of Object.values(DISK_KINDS)) {
+      for (const uuidMatches of [false, true]) {
+        const expected = uuidMatches && diskFormat === matchingFormat
+
+        test(`${kind} disk on a ${diskFormat} remote, uuidMatches=${uuidMatches} -> ${expected}`, async () => {
+          const adapter = await getAdapter({ diskFormat, remoteCompressionType: COMPRESSION })
+          const targetUuid = uniqueIdBuffer()
+          const path = await createParent(kind, uuidMatches ? targetUuid : uniqueIdBuffer())
+
+          assert.equal(await adapter.isMergeableParent(targetUuid, path), expected)
+        })
+      }
+    }
+  }
+})
+
 describe('RemoteAdapter#isMergeableParent compression resolution', { concurrency: 1 }, () => {
   test('the per-remote compressionType override wins over the adapter default', async () => {
     const adapter = await getAdapter({
@@ -205,7 +267,7 @@ describe('RemoteAdapter#isMergeableParent compression resolution', { concurrency
 
 const DEFAULT_BLOCK_SIZE = 0x00200000 // 2MB, from vhd-lib spec
 
-// minimal source disk, enough for writeVhd() to produce a real, readable VHD
+// minimal source disk, enough for writeDisk() to produce a real, readable VHD
 class MockDisk extends RandomAccessDisk {
   #size
   #blockIndexes
@@ -252,15 +314,15 @@ class MockDisk extends RandomAccessDisk {
 
 // IncrementalRemoteWriter._transfer(): the disk is written
 // through the adapter, and the *real* checkDisk() is used as the validator
-describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, () => {
+describe('RemoteAdapter#writeDisk validated by checkDisk', { concurrency: 1 }, () => {
   for (const useVhdDirectory of [false, true]) {
     const mode = useVhdDirectory ? 'directory' : 'file'
 
     test(`${mode}: writes a base disk that passes validation and reads back`, async () => {
       const adapter = await getAdapter({ useVhdDirectory })
-      const path = `${basePath}/${adapter.getVhdFileName('base')}`
+      const path = `${basePath}/${adapter.getDiskFileName('base')}`
 
-      const size = await adapter.writeVhd(path, new MockDisk(2, [0, 1], 0x42), {
+      const size = await adapter.writeDisk(path, new MockDisk(2, [0, 1], 0x42), {
         validator: tmpPath => checkDisk(handler, tmpPath),
         uuid: uniqueIdBuffer(),
       })
@@ -280,15 +342,15 @@ describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, ()
 
     test(`${mode}: writes a differencing disk chained to its parent that passes validation`, async () => {
       const adapter = await getAdapter({ useVhdDirectory })
-      const parentPath = `${basePath}/${adapter.getVhdFileName('20000101T000000Z')}`
-      const childPath = `${basePath}/${adapter.getVhdFileName('20000102T000000Z')}`
+      const parentPath = `${basePath}/${adapter.getDiskFileName('20000101T000000Z')}`
+      const childPath = `${basePath}/${adapter.getDiskFileName('20000102T000000Z')}`
       const parentUuid = uniqueIdBuffer()
 
-      await adapter.writeVhd(parentPath, new MockDisk(2, [0, 1], 0xaa), {
+      await adapter.writeDisk(parentPath, new MockDisk(2, [0, 1], 0xaa), {
         validator: tmpPath => checkDisk(handler, tmpPath),
         uuid: parentUuid,
       })
-      await adapter.writeVhd(childPath, new MockDisk(2, [1], 0xbb, { differencing: true }), {
+      await adapter.writeDisk(childPath, new MockDisk(2, [1], 0xbb, { differencing: true }), {
         validator: tmpPath => checkDisk(handler, tmpPath),
         uuid: uniqueIdBuffer(),
         parentUuid,
@@ -304,10 +366,10 @@ describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, ()
 
     test(`${mode}: a rejected validation leaves no disk behind`, async () => {
       const adapter = await getAdapter({ useVhdDirectory })
-      const path = `${basePath}/${adapter.getVhdFileName('base')}`
+      const path = `${basePath}/${adapter.getDiskFileName('base')}`
 
       await assert.rejects(
-        adapter.writeVhd(path, new MockDisk(1, [0], 0x11), {
+        adapter.writeDisk(path, new MockDisk(1, [0], 0x11), {
           validator: async () => {
             throw new Error('boom')
           },
@@ -323,3 +385,104 @@ describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, ()
     })
   }
 })
+
+// every `<hash>` entry of the shared block store
+async function listBlockStore() {
+  const hashes = []
+  for (const ab of await handler.list(BLOCK_STORE_PATH, { ignoreMissing: true })) {
+    for (const cd of await handler.list(`${BLOCK_STORE_PATH}/${ab}`)) {
+      hashes.push(...(await handler.list(`${BLOCK_STORE_PATH}/${ab}/${cd}`)))
+    }
+  }
+  return hashes
+}
+
+describe('RemoteAdapter#writeDisk on a hbd remote', { concurrency: 1 }, () => {
+  const writeHbdDisk = (adapter, path, disk, opts) =>
+    adapter.writeDisk(path, disk, {
+      validator: tmpPath => checkDisk(handler, tmpPath),
+      uuid: uniqueIdBuffer(),
+      ...opts,
+    })
+
+  test('names the disk file .hbd', async () => {
+    const adapter = await getAdapter({ diskFormat: 'hbd' })
+
+    assert.equal(adapter.getDiskFileName('20000101T000000Z'), '20000101T000000Z.hbd')
+  })
+
+  test('writes a base disk that passes validation and reads back', async () => {
+    const adapter = await getAdapter({ diskFormat: 'hbd' })
+    const path = `${basePath}/${adapter.getDiskFileName('base')}`
+    const source = new MockDisk(2, [0, 1], 0x42)
+    const diskUuid = uniqueIdBuffer()
+
+    const size = await writeHbdDisk(adapter, path, source, { uuid: diskUuid })
+
+    // what was read from the source: its 2 blocks, not its (geometry rounded) virtual size
+    assert.equal(size, 2 * DEFAULT_BLOCK_SIZE)
+    await Disposable.use(openDisposableDisk({ handler, path }), async disk => {
+      // the writer passes packed uuids, the hbd metadata holds them as strings
+      assert.equal(disk.getUuid(), uuid.stringify(diskUuid))
+      assert.equal(disk.isDifferencing(), false)
+      assert.deepEqual(disk.getBlockIndexes(), [0, 1])
+      for (const index of [0, 1]) {
+        const { data } = await disk.readBlock(index)
+        const expected = (await source.readBlock(index)).data
+        assert.deepEqual(data.subarray(0, expected.length), expected, `block ${index} should match the source`)
+      }
+    })
+  })
+
+  test('writes a differencing disk chained to its parent that passes validation', async () => {
+    const adapter = await getAdapter({ diskFormat: 'hbd' })
+    const parentPath = `${basePath}/${adapter.getDiskFileName('20000101T000000Z')}`
+    const childPath = `${basePath}/${adapter.getDiskFileName('20000102T000000Z')}`
+    const parentUuid = uniqueIdBuffer()
+
+    await writeHbdDisk(adapter, parentPath, new MockDisk(2, [0, 1], 0xaa), { uuid: parentUuid })
+    await writeHbdDisk(adapter, childPath, new MockDisk(2, [1], 0xbb, { differencing: true }), {
+      parentUuid,
+      parentPath: relativeFromFile(childPath, parentPath),
+    })
+
+    await Disposable.use(openDisposableDisk({ handler, path: childPath }), async disk => {
+      assert.equal(disk.isDifferencing(), true)
+      assert.equal(disk.getParentUuid(), uuid.stringify(parentUuid))
+      assert.equal(disk.getParentPath(), `/${parentPath}`)
+      assert.deepEqual(disk.getBlockIndexes(), [1])
+    })
+  })
+
+  test('identical content is stored once for the whole remote', async () => {
+    const adapter = await getAdapter({ diskFormat: 'hbd' })
+    const otherBasePath = `${rootPath}/vdis/${jobId}/${uniqueId()}`
+
+    await writeHbdDisk(adapter, `${basePath}/${adapter.getDiskFileName('a')}`, new MockDisk(2, [0, 1], 0x42))
+    const storeAfterFirst = await listBlockStore()
+
+    // another VDI of the same remote, same content
+    await writeHbdDisk(adapter, `${otherBasePath}/${adapter.getDiskFileName('b')}`, new MockDisk(2, [0, 1], 0x42))
+
+    assert.deepEqual((await listBlockStore()).sort(), storeAfterFirst.sort())
+  })
+
+  test('a rejected validation leaves no disk and no block behind', async () => {
+    const adapter = await getAdapter({ diskFormat: 'hbd' })
+    const path = `${basePath}/${adapter.getDiskFileName('base')}`
+
+    await assert.rejects(
+      writeHbdDisk(adapter, path, new MockDisk(1, [0], 0x11), {
+        validator: async () => {
+          throw new Error('boom')
+        },
+      }),
+      /boom/
+    )
+
+    // `data/` itself is shared with the other disks of the directory
+    assert.deepEqual(await handler.list(basePath, { ignoreMissing: true }), ['data'])
+    assert.deepEqual(await handler.list(`${basePath}/data`, { ignoreMissing: true }), [])
+    assert.deepEqual(await listBlockStore(), [])
+  })
+})

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -28,6 +28,8 @@ import { watchStreamSize } from './_watchStreamSize.mjs'
 
 import { RemoteVhdDisk, openDiskChain, openDisposableDisk } from '@xen-orchestra/backup-archive/disks'
 import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
+import { writeHbd } from '@xen-orchestra/backup-archive/disks/hashed'
+
 import { ReadAhead } from '@xen-orchestra/disk-transform'
 
 export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
@@ -71,10 +73,14 @@ export class RemoteAdapter {
   // a requirement here.
   async isMergeableParent(packedParentUid, path) {
     return await Disposable.use(openDisposableDisk({ handler: this.handler, path, ignoreBlockIndexes: true }), disk =>
-      disk.isMergeableParent(stringify(packedParentUid))
+      disk.isMergeableParent(stringify(packedParentUid), { diskFormat: this.#diskFormat })
     )
   }
 
+  get #diskFormat() {
+    return this.handler.getConfig('diskFormat')
+  }
+
   // Single funnel for all the backup deletions triggered by a user or by retention: forgets them
   // from `cache.json.gz` and records them in the journal.
   /**
@@ -218,19 +224,23 @@ export class RemoteAdapter {
   }
 
   useVhdDirectory() {
-    return this.handler.getConfig('useVhdDirectory')
+    return this.handler.getConfig('diskFormat') === 'vhdDirectory'
   }
 
   #useAlias() {
     return this.useVhdDirectory()
   }
 
-  // if we use alias on this remote, we have to name the file alias.vhd
-  getVhdFileName(baseName) {
-    if (this.#useAlias()) {
-      return `${baseName}.alias.vhd`
+  // if we use alias on this remote, we have to name the file alias.{vhd|hbd}
+  getDiskFileName(baseName) {
+    if (this.#diskFormat === 'hbd') {
+      return `${baseName}.hbd`
+    } else {
+      if (this.#useAlias()) {
+        return `${baseName}.alias.vhd`
+      }
+      return `${baseName}.vhd`
     }
-    return `${baseName}.vhd`
   }
 
   async listAllVms() {
@@ -480,33 +490,45 @@ export class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, disk, { validator = noop, writeBlockConcurrency, uuid, parentUuid, parentPath } = {}) {
+  async writeDisk(path, disk, { validator = noop, writeBlockConcurrency, uuid, parentUuid, parentPath } = {}) {
     const handler = this._handler
-
-    if (this.useVhdDirectory()) {
-      return await writeToVhdDirectory({
-        disk,
-        target: {
-          handler,
-          path,
+    const diskFormat = this.#diskFormat
+    switch (diskFormat) {
+      case 'vhd': {
+        const stream = await toVhdStream(disk, { uuid, parentUuid, parentPath })
+        const size = await this.outputStream(path, stream, {
+          validator,
+          // no checksum for VHDs, because they will be invalidated by
+          // merges and chains
+          checksum: false,
+        })
+        await validator(path)
+        return size
+      }
+      case 'vhdDirectory':
+        return await writeToVhdDirectory({
+          disk,
+          target: {
+            handler,
+            path,
+            concurrency: writeBlockConcurrency,
+            validator,
+            compression: handler.getConfig('compressionType') ?? 'brotli', // compatibility layer
+            uuid,
+            parentUuid,
+            parentPath,
+          },
+        })
+      case 'hbd':
+        return await writeHbd(handler, path, disk, {
+          uuid: stringify(uuid),
+          parentUuid: parentUuid && stringify(parentUuid),
+          parentPath,
           concurrency: writeBlockConcurrency,
           validator,
-          compression: handler.getConfig('compressionType') ?? 'brotli', // compatibility layer
-          uuid,
-          parentUuid,
-          parentPath,
-        },
-      })
-    } else {
-      const stream = await toVhdStream(disk, { uuid, parentUuid, parentPath })
-      const size = await this.outputStream(path, stream, {
-        validator,
-        // no checksum for VHDs, because they will be invalidated by
-        // merges and chains
-        checksum: false,
-      })
-      await validator(path)
-      return size
+        })
+      default:
+        throw new Error(`unknown disk format ${diskFormat} for ${path}`)
     }
   }
 

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -16,7 +16,7 @@ import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
 import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
 import { checkDisk } from './_checkDisk.mjs'
 import { packUuid } from './_packUuid.mjs'
-import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
+import { isDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
 import { VDI_FORMAT_QCOW2 } from '@xen-orchestra/xapi'
 import { diskHasData } from './_diskHasData.mjs'
 
@@ -39,7 +39,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       const vhdDir = `${vdisDir}/${srcVdiUuid}`
       try {
         const vhds = await handler.list(vhdDir, {
-          filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
+          filter: _ => _[0] !== '.' && isDisk(_),
           ignoreMissing: true,
           prependDir: true,
         })
@@ -92,7 +92,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     // parentDestPath is the chain tip = most-recent backup; its filename is stable under
     // merges (merges rename the OLDER disks), so unlike the root it maps reliably to its
     // metadata json.
-    const parentDate = basename(parentDestPath).replace(/\.(alias\.)?vhd$/, '')
+    const parentDate = basename(parentDestPath).replace(/\.(alias\.vhd|vhd|hbd)$/, '')
     let metadata
     try {
       metadata = await this._adapter.readVmBackupMetadata(`${this._vmBackupDir}/${parentDate}.json`)
@@ -225,7 +225,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${adapter.getVhdFileName(filenameDate)}`
+        }/${adapter.getDiskFileName(filenameDate)}`
     )
 
     let metadataContent = await this._isAlreadyTransferred(timestamp)
@@ -273,7 +273,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
             parentPath = relativeFromFile(path, parentDestPath)
           }
 
-          const transferred = await adapter.writeVhd(path, disk, {
+          const transferred = await adapter.writeDisk(path, disk, {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chains
             checksum: false,

@xen-orchestra/backups/docs/VM backups/hbd.md

@@ -0,0 +1,148 @@
+# Hashed disks (`.hbd`)
+
+A content addressed disk format: each block is stored once, named after the SHA-256 of its content.
+
+- [File structure on remote](#file-structure-on-remote)
+- [Files](#files)
+  - [`<datetime>.hbd`](#datetimehbd)
+  - [`hashes.<ms>.<random>.hash`](#hashesmsrandomhash)
+  - [Block file](#block-file)
+- [Dedup types](#dedup-types)
+- [Reference counting](#reference-counting)
+  - [Link limit (`EMLINK`)](#link-limit-emlink)
+- [Lifecycle](#lifecycle)
+
+## File structure on remote
+
+```
+<remote>
+├─ xo-vm-backups
+│  └─ <VM UUID>
+│     └─ vdis
+│        └─ <job UUID>
+│           └─ <VDI UUID>
+│              ├─ <YYYYMMDD>T<HHmmss>.hbd           // one per backup
+│              └─ data
+│                 └─ <disk UUID at creation>
+│                    ├─ hashes.<ms>.<random>.hash  // the BAT
+│                    └─ blocks
+│                       └─ <ab>/<cd>/<hash>         // one file per unique block
+└─ xo-block-store                                   // PER_BACKUP_REPOSITORY only
+   └─ <ab>/<cd>/<hash>
+      ├─ 0                                          // store copy
+      └─ 1, 2…                                      // overflow copies, when 0 hits the link limit
+```
+
+- `<ab>/<cd>`: the first 2 bytes of the hash, so at most 256 entries per level whatever the number of blocks
+- `<hash>`: the full hash, 64 hex characters
+- the data directory keeps the uuid the disk had when it was created: a merge changes the disk uuid, not the directory
+
+## Files
+
+### `<datetime>.hbd`
+
+```json
+{
+  "version": "0.0.1",
+  "virtualSize": 21474836480,
+  "blockSize": 2097152,
+  "uuid": "<disk UUID>",
+  "parentUuid": "<parent UUID>",
+  "parentPath": "./<parent datetime>.hbd",
+  "dedupType": "PER_BACKUP_REPOSITORY",
+  "localBlocksPath": "data/<disk UUID at creation>/blocks/",
+  "hashesPath": "data/<disk UUID at creation>/hashes.<ms>.<random>.hash"
+}
+```
+
+- paths are relative to the `.hbd` file, and must stay inside the disk data directory
+- `parentUuid` / `parentPath` are absent on a full disk
+
+### `hashes.<ms>.<random>.hash`
+
+The BAT: a flat binary file, one 32-byte SHA-256 per block index.
+
+```
+offset  index * 32
+size    ceil(virtualSize / blockSize) * 32
+empty   32 zero bytes
+```
+
+- a new file at each flush, never overwritten: the `.hbd` points at the current one
+- any other `hashes.*.hash` in the same directory is a leftover of a crash
+
+### Block file
+
+```
+0..511    header
+  0..3      magic "HBD\0"
+  4         codec (0 raw, 1 brotli)
+  8..15     verifiedAt (ms, 0 = never)
+  16..47    SHA-256 of the payload
+  48..51    payloadLength (uint32 LE, never 0)
+  52..511   reserved (zero)
+512..     payload (payloadLength bytes)
+```
+
+## Dedup types
+
+| `dedupType`             | Blocks                                                            | Needs hard links |
+| ----------------------- | ----------------------------------------------------------------- | ---------------- |
+| `PER_DISK`              | real files in the disk `blocks/`                                  | no               |
+| `PER_BACKUP_REPOSITORY` | hard links to `xo-block-store`, shared by all disks of the remote | yes              |
+
+- `PER_DISK` for remotes without hard links (S3, Azure)
+- the store location is given by the caller, never read from the `.hbd`
+  - `BLOCK_STORE_PATH` (`xo-block-store`) by default
+
+## Reference counting
+
+No counter file: the link count of the inode is the count.
+
+```
+xo-block-store/<h>/0  ─┐
+diskA/blocks/<h>      ─┼──▶ [ inode ]  nlink = 3
+diskB/blocks/<h>      ─┘
+```
+
+- `nlink - 1` = number of disks using the block
+- a store file at `nlink = 1` is used by no disk: the GC scan (`scanBlockStore`) removes it
+  - maintenance only, run by an admin with no backup job running: `xo-disk-cli scan-store <remote url> [--remove] [--verify]`
+  - after a backup cleanup: a leftover temp file (`blocks/.tmp/`) keeps its store file at `nlink = 2` until the disk `clean()` removes it
+- reads always go through the disk own link, never through the store
+
+### Link limit (`EMLINK`)
+
+An inode accepts a limited number of links (ext4: 65000, XFS: 65535). A common block, the all-zero one first, can reach it on a large remote.
+
+```
+xo-block-store/<h>/0  ─┐
+disk 1 … disk 64999   ─┴──▶ [ inode #1 ]  nlink = 65000 (full)
+
+xo-block-store/<h>/1  ─┐
+disk 65000            ─┴──▶ [ inode #2 ]  nlink = 2
+```
+
+- `link` fails with `EMLINK` on `0`: try the next copies `1`, `2`…
+- all full: publish a new copy `N+1`, then link to it
+- each copy is its own inode, so its count is independent
+- releasing a block removes the copy that drops to `nlink = 1`, not always `0`
+
+## Lifecycle
+
+| Step   | What happens                                                                                       |
+| ------ | -------------------------------------------------------------------------------------------------- |
+| write  | link from the store, or publish a new store file (temp file, then `link`)                          |
+| flush  | new BAT, then `.hbd`, then release the blocks no index uses any more, then remove the previous BAT |
+| merge  | parent takes the child blocks and uuid, then is renamed to the child path                          |
+| unlink | release every block of the BAT, then remove the data directory and the `.hbd`                      |
+| clean  | `clean()` removes leftover BATs, `scanBlockStore` removes store files at `nlink = 1`               |
+
+- write: `writeHbd(handler, path, source, { uuid, parentUuid, parentPath })`, called by `RemoteAdapter.writeDisk` when the remote `diskFormat` is `hbd`
+  - on failure, waits for the pending block writes, then `unlink()`: no reference is left behind
+- merge: the child is a `HashedDiskChain` (opened by `openDiskChain`), never a bare disk
+  - `mergeBlock` links `child.getBlockPath(index)` into the parent, skipped when the parent already holds the same hash
+- clean: `clean()` is the disk hook called by `RemoteDiskLineage`; it also removes leftover temp files (`blocks/.tmp/`)
+
+- releasing a block: unlink the disk link, and the store file too if this disk was its last user
+- a crash at any step leaves a readable disk: only leftovers, cleaned later

@xen-orchestra/disk-cli/.USAGE.md

@@ -1,10 +1,12 @@
 ```
 $ xo-disk-cli --help
 Usage: xo-disk-cli <command> <handler-url> <path> [options]
+       xo-disk-cli scan-store <handler-url> [--remove] [--verify] [--verbose]
 
 Commands:
   info       Show disk info (virtual size, uid, parent uid, block size); --chain shows the full parent chain, --size adds the size on disk
   list       List all disks at a path and display their properties in a table; --size adds the size on disk
+  scan-store Maintenance of the hbd block store: reports the unused blocks, --remove deletes them, --verify rereads every block. Run with no backup job using the remote, after a backup cleanup
   transform  Convert a disk to another format and write to stdout (raw | vhd | qcow2)
 ```
 
@@ -97,6 +99,28 @@ $ xo-disk-cli list file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/
 └──────────────┴──────────────────────────────────────┴─────────────┴──────────────┴─────────────┴──────────────────────────────────────┘
 ```
 
+### `scan-store`
+
+Maintenance of the block store (`xo-block-store/`) of a remote holding hbd (deduplicated) disks. Never run by a backup.
+
+```
+xo-disk-cli scan-store <handler-url> [--remove] [--verify] [--verbose]
+```
+
+- reports the store blocks no disk uses any more (link count 1); `--remove` deletes them
+- `--verify` rereads every block still in use and checks it against its hash; a corrupted block is reported, never repaired nor removed
+- `--verbose` prints every unused block
+- run it with no backup job using the remote, after a backup cleanup: a temp file left by an interrupted backup keeps its block in use until the cleanup removes it
+
+```
+$ xo-disk-cli scan-store file:///mnt/backups
+unused store blocks: 12 found (dry run, use --remove to delete them)
+
+$ xo-disk-cli scan-store file:///mnt/backups --remove --verify
+unused store blocks: 12 removed
+corrupted store blocks: 0
+```
+
 ### `transform`
 
 Convert a disk (or its full parent chain if differencing) to the specified format and write the result to stdout. Redirect to a file to save the output.

@xen-orchestra/disk-cli/README.md

@@ -19,10 +19,12 @@ npm install --global @xen-orchestra/disk-cli
 ```
 $ xo-disk-cli --help
 Usage: xo-disk-cli <command> <handler-url> <path> [options]
+       xo-disk-cli scan-store <handler-url> [--remove] [--verify] [--verbose]
 
 Commands:
   info       Show disk info (virtual size, uid, parent uid, block size); --chain shows the full parent chain, --size adds the size on disk
   list       List all disks at a path and display their properties in a table; --size adds the size on disk
+  scan-store Maintenance of the hbd block store: reports the unused blocks, --remove deletes them, --verify rereads every block. Run with no backup job using the remote, after a backup cleanup
   transform  Convert a disk to another format and write to stdout (raw | vhd | qcow2)
 ```
 
@@ -115,6 +117,28 @@ $ xo-disk-cli list file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/
 └──────────────┴──────────────────────────────────────┴─────────────┴──────────────┴─────────────┴──────────────────────────────────────┘
 ```
 
+### `scan-store`
+
+Maintenance of the block store (`xo-block-store/`) of a remote holding hbd (deduplicated) disks. Never run by a backup.
+
+```
+xo-disk-cli scan-store <handler-url> [--remove] [--verify] [--verbose]
+```
+
+- reports the store blocks no disk uses any more (link count 1); `--remove` deletes them
+- `--verify` rereads every block still in use and checks it against its hash; a corrupted block is reported, never repaired nor removed
+- `--verbose` prints every unused block
+- run it with no backup job using the remote, after a backup cleanup: a temp file left by an interrupted backup keeps its block in use until the cleanup removes it
+
+```
+$ xo-disk-cli scan-store file:///mnt/backups
+unused store blocks: 12 found (dry run, use --remove to delete them)
+
+$ xo-disk-cli scan-store file:///mnt/backups --remove --verify
+unused store blocks: 12 removed
+corrupted store blocks: 0
+```
+
 ### `transform`
 
 Convert a disk (or its full parent chain if differencing) to the specified format and write the result to stdout. Redirect to a file to save the output.

@xen-orchestra/disk-cli/src/commands/scanStore.mts

@@ -0,0 +1,35 @@
+import Disposable from 'promise-toolbox/Disposable'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { scanBlockStore } from '@xen-orchestra/backup-archive/disks/hashed'
+
+/**
+ * Maintenance of the block store of a hbd remote: never run by a backup.
+ *
+ * Without --remove, only reports. To run with no backup job using the remote.
+ */
+export async function scanStoreCommand(handlerUrl: string, _path: string, extraArgs: string[]): Promise<void> {
+  const remove = extraArgs.includes('--remove')
+  const verify = extraArgs.includes('--verify')
+  const verbose = extraArgs.includes('--verbose')
+
+  await Disposable.use(getSyncedHandler({ url: handlerUrl }), async handler => {
+    let corrupted = 0
+    const { removed } = await scanBlockStore(handler, {
+      remove,
+      verify,
+      logWarn: (message, data: { path?: string; error?: Error } = {}) => {
+        corrupted++
+        console.error(`${message}: ${data.path} (${data.error?.message ?? 'unknown error'})`)
+      },
+      logDebug: verbose
+        ? (message, data: { path?: string } = {}) => console.error(`${message}: ${data.path}`)
+        : () => {},
+    })
+
+    const orphans = remove ? `${removed} removed` : `${removed} found (dry run, use --remove to delete them)`
+    console.log(`unused store blocks: ${orphans}`)
+    if (verify) {
+      console.log(`corrupted store blocks: ${corrupted}`)
+    }
+  })
+}

@xen-orchestra/disk-cli/src/declarations.d.ts

@@ -46,6 +46,21 @@ declare module '@xen-orchestra/backup-archive/disks/openDiskChain.mjs' {
   }): Promise<RandomAccessDisk>
 }
 
+declare module '@xen-orchestra/backup-archive/disks/hashed' {
+  import type { RemoteHandler } from '@xen-orchestra/fs'
+
+  export function scanBlockStore(
+    handler: RemoteHandler,
+    opts?: {
+      remove?: boolean
+      verify?: boolean
+      logInfo?: (message: string, data?: object) => void
+      logWarn?: (message: string, data?: object) => void
+      logDebug?: (message: string, data?: object) => void
+    }
+  ): Promise<{ removed: number }>
+}
+
 declare module '@xen-orchestra/backup-archive/disks' {
   import type { RemoteHandler } from '@xen-orchestra/fs'
   import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'

@xen-orchestra/disk-cli/src/index.mts

@@ -2,6 +2,7 @@
 
 import { infoCommand } from './commands/info.mjs'
 import { listCommand } from './commands/list.mjs'
+import { scanStoreCommand } from './commands/scanStore.mjs'
 import { transformCommand } from './commands/transform.mjs'
 
 type CommandFn = (handlerUrl: string, path: string, extraArgs: string[]) => Promise<void>
@@ -9,13 +10,19 @@ type CommandFn = (handlerUrl: string, path: string, extraArgs: string[]) => Prom
 const COMMANDS: Record<string, CommandFn> = {
   info: infoCommand,
   list: listCommand,
+  'scan-store': scanStoreCommand,
   transform: transformCommand,
 }
 
+// commands working on the whole remote: no <path>
+const REMOTE_COMMANDS = new Set(['scan-store'])
+
 function showHelp(exitCode: number): never {
   const commandDescriptions: Record<string, string> = {
     info: 'Show disk info (virtual size, uid, parent uid, block size); --chain shows the full parent chain, --size adds the size on disk',
     list: 'List all disks at a path and display their properties in a table; --size adds the size on disk',
+    'scan-store':
+      'Maintenance of the hbd block store: reports the unused blocks, --remove deletes them, --verify rereads every block. Run with no backup job using the remote, after a backup cleanup',
     transform: 'Convert a disk to another format and write to stdout (raw | vhd | qcow2)',
   }
 
@@ -24,6 +31,7 @@ function showHelp(exitCode: number): never {
     .join('\n')
 
   process.stdout.write(`Usage: xo-disk-cli <command> <handler-url> <path> [options]
+       xo-disk-cli scan-store <handler-url> [--remove] [--verify] [--verbose]
 
 Commands:
 ${commandList}
@@ -33,20 +41,25 @@ Examples:
   xo-disk-cli info      file:///mnt/backups /xo-vm-backups/uuid/disk.alias.vhd --chain --size
   xo-disk-cli list      file:///mnt/backups /xo-vm-backups/uuid/
   xo-disk-cli list      file:///mnt/backups /xo-vm-backups/uuid/ --size
+  xo-disk-cli scan-store file:///mnt/backups
+  xo-disk-cli scan-store file:///mnt/backups --remove --verify
   xo-disk-cli transform file:///mnt/backups /xo-vm-backups/uuid/disk.vhd raw > disk.img
   xo-disk-cli transform file:///mnt/backups /xo-vm-backups/uuid/disk.vhd qcow2 > disk.qcow2
 `)
   process.exit(exitCode)
 }
 
-const [, , command, handlerUrl, diskPath, ...extraArgs] = process.argv
+const [, , command, handlerUrl, ...args] = process.argv
 
 if (!command || command === '--help' || command === '-h') {
   showHelp(0)
 }
 
-if (!handlerUrl || !diskPath) {
-  console.error('Error: handler-url and disk-path are required\n')
+const needsPath = !REMOTE_COMMANDS.has(command)
+const [diskPath, ...extraArgs] = needsPath ? args : ['', ...args]
+
+if (!handlerUrl || (needsPath && !diskPath)) {
+  console.error(needsPath ? 'Error: handler-url and disk-path are required\n' : 'Error: handler-url is required\n')
   showHelp(1)
 }
 

@xen-orchestra/disk-cli/src/test/commands/scanStore.test.mts

@@ -0,0 +1,130 @@
+/**
+ * Tests for the `scan-store` command.
+ *
+ * Focus:
+ *  - Dry run by default: --remove and --verify are only passed when asked.
+ *  - The summary tells removed from found, and counts the corrupted blocks.
+ *  - Handler is always disposed, even when the scan throws.
+ *
+ * Design: module mocks are registered once at the top level. currentScanImpl is
+ * reset in beforeEach and set per-test.
+ */
+import { describe, test, mock, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+type ScanOpts = {
+  remove?: boolean
+  verify?: boolean
+  logWarn?: (message: string, data?: object) => void
+  logDebug?: (message: string, data?: object) => void
+}
+
+const handlerDispose = mock.fn(() => Promise.resolve())
+const mockHandler = {}
+
+// Records the options each scanBlockStore call received.
+const scanOpts: ScanOpts[] = []
+
+// Reassigned per-test to control what scanBlockStore does.
+let currentScanImpl: (opts: ScanOpts) => Promise<{ removed: number }> = async () => ({ removed: 0 })
+
+mock.module('@xen-orchestra/fs', {
+  namedExports: {
+    getSyncedHandler: () => Promise.resolve({ value: mockHandler, dispose: handlerDispose }),
+  },
+})
+
+mock.module('@xen-orchestra/backup-archive/disks/hashed', {
+  namedExports: {
+    scanBlockStore: (handler: unknown, opts: ScanOpts) => {
+      assert.equal(handler, mockHandler)
+      scanOpts.push(opts)
+      return currentScanImpl(opts)
+    },
+  },
+})
+
+const { scanStoreCommand } = await import('../../commands/scanStore.mjs')
+
+// Captures console.log and console.error without polluting test output.
+async function capture(fn: () => Promise<void>): Promise<{ out: string; err: string }> {
+  const out: string[] = []
+  const err: string[] = []
+  const { log, error } = console
+  console.log = (...args: unknown[]) => out.push(args.map(String).join(' '))
+  console.error = (...args: unknown[]) => err.push(args.map(String).join(' '))
+  try {
+    await fn()
+  } finally {
+    console.log = log
+    console.error = error
+  }
+  return { out: out.join('\n'), err: err.join('\n') }
+}
+
+beforeEach(() => {
+  scanOpts.length = 0
+  handlerDispose.mock.resetCalls()
+  currentScanImpl = async () => ({ removed: 0 })
+})
+
+describe('scanStoreCommand', () => {
+  test('is a dry run by default', async () => {
+    currentScanImpl = async () => ({ removed: 3 })
+
+    const { out } = await capture(() => scanStoreCommand('file:///backups', '', []))
+
+    assert.equal(scanOpts.length, 1)
+    assert.equal(scanOpts[0].remove, false)
+    assert.equal(scanOpts[0].verify, false)
+    assert.match(out, /unused store blocks: 3 found \(dry run/)
+    assert.doesNotMatch(out, /corrupted/)
+    assert.equal(handlerDispose.mock.callCount(), 1)
+  })
+
+  test('--remove deletes the unused blocks', async () => {
+    currentScanImpl = async () => ({ removed: 2 })
+
+    const { out } = await capture(() => scanStoreCommand('file:///backups', '', ['--remove']))
+
+    assert.equal(scanOpts[0].remove, true)
+    assert.match(out, /unused store blocks: 2 removed/)
+  })
+
+  test('--verify counts and reports the corrupted blocks', async () => {
+    currentScanImpl = async ({ logWarn }) => {
+      logWarn!('corrupted store block', { path: '/xo-block-store/ab/cd/abcd/0', error: new Error('hash mismatch') })
+      logWarn!('corrupted store block', { path: '/xo-block-store/ef/01/ef01/0', error: new Error('bad magic') })
+      return { removed: 0 }
+    }
+
+    const { out, err } = await capture(() => scanStoreCommand('file:///backups', '', ['--verify']))
+
+    assert.equal(scanOpts[0].verify, true)
+    assert.match(out, /corrupted store blocks: 2/)
+    assert.match(err, /abcd\/0 \(hash mismatch\)/)
+    assert.match(err, /ef01\/0 \(bad magic\)/)
+  })
+
+  test('--verbose prints every unused block', async () => {
+    currentScanImpl = async ({ logDebug }) => {
+      logDebug!('orphan store file', { path: '/xo-block-store/ab/cd/abcd/0' })
+      return { removed: 1 }
+    }
+
+    const quiet = await capture(() => scanStoreCommand('file:///backups', '', []))
+    const verbose = await capture(() => scanStoreCommand('file:///backups', '', ['--verbose']))
+
+    assert.equal(quiet.err, '')
+    assert.match(verbose.err, /orphan store file: \/xo-block-store\/ab\/cd\/abcd\/0/)
+  })
+
+  test('disposes the handler when the scan throws', async () => {
+    currentScanImpl = async () => {
+      throw new Error('EACCES')
+    }
+
+    await assert.rejects(() => scanStoreCommand('file:///backups', '', ['--remove']), /EACCES/)
+    assert.equal(handlerDispose.mock.callCount(), 1)
+  })
+})

@xen-orchestra/fs/src/abstract.js

@@ -16,6 +16,13 @@ import { createChecksumStream, validChecksumOfReadStream } from './checksum'
 import { DEFAULT_ENCRYPTION_ALGORITHM, UNENCRYPTED_ALGORITHM, _getEncryptor } from './_encryptor'
 
 const { info, warn } = createLogger('xo:fs:abstract')
+export class NotImplementedError extends Error {
+  constructor(message = 'Not implemented') {
+    super(message)
+    this.name = 'NotImplementedError'
+    this.code = 'ENOTIMP'
+  }
+}
 
 const checksumFile = file => file + '.checksum'
 const computeRate = (hrtime, size) => {
@@ -29,11 +36,18 @@ const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
 const ENCRYPTION_DESC_FILENAME = 'encryption.json'
 const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
 
+// value of a remote option the remote URL does not set
+const CONFIG_DEFAULTS = {
+  diskFormat: 'vhd',
+}
+
 const WITH_LIMIT = [
   'closeFile',
   'copy',
   'getInfo',
   'getSizeOnDisk',
+  'getLinkCount',
+  'link',
   'list',
   'mkdir',
   'openFile',
@@ -52,7 +66,9 @@ const WITH_RETRY = {
   _closeFile: {},
   _copy: {},
   _getInfo: {},
+  _getLinkCount: {},
   _getSize: {},
+  _link: {},
   _list: {},
   _mkdir: {},
   _openFile: {},
@@ -78,7 +94,9 @@ const WITH_TIMEOUT = [
   '_closeFile',
   '_createReadStream',
   '_getInfo',
+  '_getLinkCount',
   '_getSize',
+  '_link',
   '_list',
   '_rename',
   '_copy',
@@ -123,6 +141,10 @@ class PrefixWrapper {
     return entries
   }
 
+  async link(existingPath, newPath) {
+    return this._handler.link(this._resolve(existingPath), this._resolve(newPath))
+  }
+
   rename(oldPath, newPath) {
     return this._handler.rename(this._resolve(oldPath), this._resolve(newPath))
   }
@@ -145,7 +167,12 @@ export default class RemoteHandlerAbstract {
   }
 
   getConfig(key) {
-    return this._remote[key]
+    const value = this._remote[key]
+    if (value === undefined && key === 'diskFormat') {
+      // handle legacy
+      return this._remote.useVhdDirectory ? 'vhdDirectory' : 'vhd'
+    }
+    return value ?? CONFIG_DEFAULTS[key]
   }
 
   constructor(remote, options = {}) {
@@ -169,7 +196,19 @@ export default class RemoteHandlerAbstract {
   }
 
   _conditionRetry(error) {
-    return !['EEXIST', 'EISDIR', 'ENOTEMPTY', 'ENOENT', 'ENOTDIR', 'SystemInUse', 'ERR_ASSERTION'].includes(error?.code)
+    return ![
+      'EEXIST',
+      'EISDIR',
+      'EMLINK',
+      'ENOTEMPTY',
+      'ENOENT',
+      'ENOTDIR',
+      'ENOTIMP',
+      'ENOTSUP',
+      'EPERM',
+      'SystemInUse',
+      'ERR_ASSERTION',
+    ].includes(error?.code)
   }
 
   #applySafeGuards(options) {
@@ -227,7 +266,7 @@ export default class RemoteHandlerAbstract {
   // - some preprocessing may be applied on parameters that should not be done multiple times (e.g. prefixing paths)
 
   get type() {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   addPrefix(prefix) {
@@ -345,10 +384,36 @@ export default class RemoteHandlerAbstract {
     return this.getSizeOnDisk(file)
   }
 
+  async __getLinkCount(file) {
+    return this._getLinkCount(normalizePath(file)).catch(error => {
+      if (error.code === 'ENOENT') {
+        return 0
+      }
+      throw error
+    })
+  }
+
   async getSizeOnDisk(file) {
     return this._getSize(typeof file === 'string' ? normalizePath(file) : file)
   }
 
+  async #link(existingPath, newPath, createTree = true) {
+    try {
+      await this._link(existingPath, newPath)
+    } catch (error) {
+      // ENOENT can be a missing target directory OR a missing source
+      if (error.code === 'ENOENT' && createTree) {
+        await this._mktree(dirname(newPath))
+        return this.#link(existingPath, newPath, false)
+      }
+      throw error
+    }
+  }
+
+  async __link(existingPath, newPath) {
+    return this.#link(normalizePath(existingPath), normalizePath(newPath))
+  }
+
   async __list(dir, { filter, ignoreMissing = false, prependDir = false } = {}) {
     try {
       const virtualDir = normalizePath(dir)
@@ -608,7 +673,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _createOutputStream(file, { dirMode, ...options } = {}) {
@@ -625,13 +690,13 @@ export default class RemoteHandlerAbstract {
   }
 
   async _createReadStream(file, options) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   // createWriteStream takes highWaterMark as option even if it's not documented.
   // Source: https://stackoverflow.com/questions/55026306/how-to-set-writeable-highwatermark
   async _createWriteStream(file, options) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   // called to finalize the remote
@@ -645,16 +710,24 @@ export default class RemoteHandlerAbstract {
     return () => Promise.resolve()
   }
 
+  async _getLinkCount(path) {
+    throw new NotImplementedError()
+  }
+
   async _getSize(file) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
+  }
+
+  async _link(existingPath, newPath) {
+    throw new NotImplementedError()
   }
 
   async _list(dir) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _mkdir(dir) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _mktree(dir, { mode } = {}) {
@@ -671,7 +744,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async _openFile(path, flags) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _outputFile(file, data, { dirMode, flags }) {
@@ -709,7 +782,7 @@ export default class RemoteHandlerAbstract {
   }
 
   _read(file, buffer, position) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   _readFile(file, options) {
@@ -717,14 +790,14 @@ export default class RemoteHandlerAbstract {
   }
 
   async _rename(oldPath, newPath) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
   async _copy(oldPath, newPath) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _rmdir(dir) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _rmtree(dir) {
@@ -764,7 +837,7 @@ export default class RemoteHandlerAbstract {
   async _sync() {}
 
   async _unlink(file) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _write(file, buffer, position) {
@@ -782,11 +855,11 @@ export default class RemoteHandlerAbstract {
   }
 
   async _writeFd(fd, buffer, position) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   async _writeFile(file, data, options) {
-    throw new Error('Not implemented')
+    throw new NotImplementedError()
   }
 
   get isEncrypted() {

@xen-orchestra/fs/src/abstract.test.js

@@ -6,7 +6,7 @@ import { DEFAULT_ENCRYPTION_ALGORITHM, _getEncryptor } from './_encryptor'
 import { Disposable, pFromCallback, TimeoutError } from 'promise-toolbox'
 import { getSyncedHandler } from '.'
 import { rimraf } from 'rimraf'
-import AbstractHandler from './abstract'
+import AbstractHandler, { NotImplementedError } from './abstract'
 import fs from 'fs-extra'
 import tmp from 'tmp'
 
@@ -58,6 +58,25 @@ describe('closeFile()', () => {
   })
 })
 
+describe('NotImplementedError', () => {
+  it('is thrown by an operation the handler does not implement', async () => {
+    const testHandler = new TestHandler()
+
+    await assert.rejects(testHandler.link('a', 'b'), error => {
+      assert.ok(error instanceof NotImplementedError)
+      assert.equal(error.code, 'ENOTIMP')
+      return true
+    })
+  })
+
+  it('is never retried', () => {
+    const testHandler = new TestHandler()
+
+    assert.equal(testHandler._conditionRetry(new NotImplementedError()), false)
+    assert.equal(testHandler._conditionRetry(new Error('transient')), true, 'an error without code still is')
+  })
+})
+
 describe('getConfig()', () => {
   it('is undefined when not set on the remote', () => {
     assert.equal(new TestHandler().getConfig('compressionType'), undefined)
@@ -66,6 +85,18 @@ describe('getConfig()', () => {
   it('reads the remote-level compressionType override', () => {
     assert.equal(new TestHandler({ compressionType: 'brotli' }).getConfig('compressionType'), 'brotli')
   })
+
+  it('defaults diskFormat to vhd when the remote does not set it', () => {
+    assert.equal(new TestHandler().getConfig('diskFormat'), 'vhd')
+  })
+
+  it('defaults useVhdDirectory is undefined', () => {
+    assert.equal(new TestHandler().getConfig('useVhdDirectory'), undefined)
+  })
+
+  it('reads the remote-level diskFormat', () => {
+    assert.equal(new TestHandler({ diskFormat: 'hbd' }).getConfig('diskFormat'), 'hbd')
+  })
 })
 
 describe('getInfo()', () => {

@xen-orchestra/fs/src/azure.js

@@ -20,8 +20,8 @@ export default class AzureHandler extends RemoteHandlerAbstract {
   #containerClient
 
   getConfig(key) {
-    if (key === 'useVhdDirectory') {
-      return true // compatibility layer
+    if (key === 'diskFormat') {
+      return 'vhdDirectory'
     }
     return super.getConfig(key)
   }

@xen-orchestra/fs/src/diskFormat.test.js

@@ -0,0 +1,41 @@
+import { describe, it } from 'node:test'
+import { strict as assert } from 'assert'
+
+import { getHandler } from '.'
+
+// building a handler does not connect: these only read the parsed remote URL
+const diskFormatOf = url => getHandler({ url }).getConfig('diskFormat')
+
+const AZURE_KEY = Buffer.from('key').toString('base64')
+
+describe('getConfig("diskFormat")', () => {
+  describe('file', () => {
+    it('defaults to vhd', () => {
+      assert.equal(diskFormatOf('file:///backups'), 'vhd')
+    })
+
+    it('reads a legacy useVhdDirectory remote as vhdDirectory', () => {
+      assert.equal(diskFormatOf('file:///backups?useVhdDirectory=true'), 'vhdDirectory')
+    })
+
+    it('reads the diskFormat option', () => {
+      assert.equal(diskFormatOf('file:///backups?diskFormat=%22hbd%22'), 'hbd')
+    })
+  })
+
+  // object storage only holds VHD directories, whatever the URL says
+  describe('s3', () => {
+    it('is always vhdDirectory', () => {
+      assert.equal(diskFormatOf('s3://user:pass@s3.example.org/bucket/dir'), 'vhdDirectory')
+      assert.equal(diskFormatOf('s3://user:pass@s3.example.org/bucket/dir?diskFormat=%22hbd%22'), 'vhdDirectory')
+    })
+  })
+
+  describe('azure', () => {
+    it('is always vhdDirectory', () => {
+      const url = `azure://account:${AZURE_KEY}@account.blob.core.windows.net/container/dir`
+      assert.equal(diskFormatOf(url), 'vhdDirectory')
+      assert.equal(diskFormatOf(`${url}?diskFormat=%22hbd%22`), 'vhdDirectory')
+    })
+  })
+})

@xen-orchestra/fs/src/fs.test.js

@@ -110,6 +110,30 @@ handlers.forEach(url => {
       })
     })
 
+    // no hard links on object storage
+    describe('#getLinkCount()', () => {
+      it('counts one link for a plain file', { skip: skipFsNotInAzure() }, async () => {
+        await handler.outputFile('file', TEST_DATA)
+        assert.equal(await handler.getLinkCount('file'), 1)
+      })
+
+      it('counts every name sharing the file', { skip: skipFsNotInAzure() }, async () => {
+        await handler.outputFile('file', TEST_DATA)
+        await handler.link('file', 'dir/other')
+
+        assert.equal(await handler.getLinkCount('file'), 2)
+        assert.equal(await handler.getLinkCount('dir/other'), 2)
+      })
+
+      it('returns 0 for a missing file, like after its last unlink', { skip: skipFsNotInAzure() }, async () => {
+        assert.equal(await handler.getLinkCount('missing'), 0)
+
+        await handler.outputFile('file', TEST_DATA)
+        await handler.unlink('file')
+        assert.equal(await handler.getLinkCount('file'), 0)
+      })
+    })
+
     describe('#list()', () => {
       it(`should list the content of folder`, async () => {
         await handler.outputFile('file', TEST_DATA)

@xen-orchestra/fs/src/index.js

@@ -7,6 +7,7 @@ import RemoteHandlerS3 from './s3'
 import RemoteHandlerSmb from './smb'
 import RemoteHandlerAzure from './azure'
 export { DEFAULT_ENCRYPTION_ALGORITHM, UNENCRYPTED_ALGORITHM, isLegacyEncryptionAlgorithm } from './_encryptor'
+export { NotImplementedError } from './abstract'
 
 const HANDLERS = {
   file: RemoteHandlerLocal,
@@ -38,5 +39,4 @@ export const getSyncedHandler = async (...opts) => {
   }
 }
 
-
-export { default as RemoteHandlerAbstract } from './abstract.js'
+export { default as RemoteHandlerAbstract } from './abstract.js'

@xen-orchestra/fs/src/local.js

@@ -129,11 +129,20 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     return info
   }
 
+  async _getLinkCount(path) {
+    const stats = await this.#addSyncStackTrace(fs.stat, this.getFilePath(path))
+    return stats.nlink
+  }
+
   async _getSize(file) {
     const stats = await this.#addSyncStackTrace(fs.stat, this.getFilePath(typeof file === 'string' ? file : file.path))
     return stats.size
   }
 
+  async _link(existingPath, newPath) {
+    return this.#addSyncStackTrace(fs.link, this.getFilePath(existingPath), this.getFilePath(newPath))
+  }
+
   async _list(dir) {
     return this.#addSyncStackTrace(fs.readdir, this.getFilePath(dir))
   }

@xen-orchestra/fs/src/path.js

@@ -11,6 +11,16 @@ export { basename, dirname, join }
 // - no duplicate slashes
 export const normalize = path => resolve('/', path)
 
+// true if `path` is `dir` itself, or is contained in `dir`
+export function isInDir(path, dir) {
+  const normalizedDir = normalize(dir)
+  const normalizedPath = normalize(path)
+
+  return (
+    normalizedPath === normalizedDir || normalizedPath.startsWith(normalizedDir === '/' ? '/' : normalizedDir + '/')
+  )
+}
+
 export function split(path) {
   const parts = normalize(path).split(sep)
 

@xen-orchestra/fs/src/path.test.js

@@ -1,7 +1,7 @@
 import { describe, it } from 'node:test'
 import { strict as assert } from 'assert'
 
-import { relativeFromFile } from './path.js'
+import { isInDir, relativeFromFile } from './path.js'
 
 describe('relativeFromFile()', function () {
   for (const [title, args] of Object.entries({
@@ -15,3 +15,25 @@ describe('relativeFromFile()', function () {
     })
   }
 })
+
+describe('isInDir()', function () {
+  for (const [title, [args, expected]] of Object.entries({
+    'the dir itself': [['/foo/bar', '/foo/bar'], true],
+    'a direct child': [['/foo/bar/baz.vhd', '/foo/bar'], true],
+    'a deep child': [['/foo/bar/baz/qux.vhd', '/foo/bar'], true],
+    'a sibling with a longer name': [['/foo/bar-baz/qux.vhd', '/foo/bar'], false],
+    'a sibling': [['/foo/baz/qux.vhd', '/foo/bar'], false],
+    'the parent dir': [['/foo', '/foo/bar'], false],
+    'a relative path': [['foo/bar/baz.vhd', '/foo/bar'], true],
+    'a relative dir': [['/foo/bar/baz.vhd', 'foo/bar'], true],
+    'a trailing slash on the dir': [['/foo/bar/baz.vhd', '/foo/bar/'], true],
+    'duplicate slashes': [['//foo///bar//baz.vhd', '/foo/bar'], true],
+    'a dot segment': [['/foo/bar/./baz.vhd', '/foo/bar'], true],
+    'an escaping dot dot segment': [['/foo/bar/../baz.vhd', '/foo/bar'], false],
+    'the root dir': [['/foo/bar.vhd', '/'], true],
+  })) {
+    it('works with ' + title, function () {
+      assert.equal(isInDir(...args), expected)
+    })
+  }
+})

@xen-orchestra/fs/src/s3.js

@@ -48,8 +48,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
   #minPartSize
 
   getConfig(key) {
-    if (key === 'useVhdDirectory') {
-      return true // compatibility layer
+    if (key === 'diskFormat') {
+      return 'vhdDirectory'
     }
     return super.getConfig(key)
   }

@xen-orchestra/fs/src/types/fs.mts

@@ -2,11 +2,17 @@ import type { Readable, Writable } from 'stream'
 
 // ─── Supporting types ────────────────────────────────────────────────────────
 
+/** thrown when a handler does not support an operation, never retried */
+export declare class NotImplementedError extends Error {
+  code: 'ENOTIMP'
+}
+
 export interface RemoteInfo {
   url: string
   encryptionKey?: string
   immutable?: boolean
-  useVhdDirectory?: boolean
+  useVhdDirectory?: boolean // legacy, read through diskFormat
+  diskFormat?: DiskFormat
   compressionType?: string
   [key: string]: unknown
 }
@@ -112,6 +118,8 @@ export interface LockDisposer {
   dispose: () => Promise<void>
 }
 
+export type DiskFormat = 'vhd' | 'vhdDirectory' | 'hbd'
+
 // ─── Abstract base class ─────────────────────────────────────────────────────
 
 export abstract class RemoteHandlerAbstract {
@@ -121,17 +129,18 @@ export abstract class RemoteHandlerAbstract {
   abstract get encryptionAlgorithm(): string
   abstract sync(): Promise<void>
   abstract forget(): Promise<void>
+  abstract link(existingPath: string, newPath: string): Promise<void>
   abstract list(dir: string, options?: ListOptions): Promise<string[]>
   abstract mkdir(dir: string, options?: MkdirOptions): Promise<void>
   abstract mktree(dir: string, options?: MkdirOptions): Promise<void>
   abstract rmdir(dir: string): Promise<void>
   abstract rmtree(dir: string): Promise<void>
   abstract createReadStream(
-    file: FileArg,
+    file: FileDescriptor,
     options?: ReadStreamOptions
   ): Promise<Readable & { length?: number; maxStreamLength?: number }>
 
-  abstract readFile(file: string, options?: ReadFileOptions): Promise<string>
+  abstract readFile(file: string, options?: ReadFileOptions): Promise<Buffer>
 
   abstract read(file: FileDescriptor, buffer: Buffer, position?: number): Promise<ReadResult>
 
@@ -139,7 +148,7 @@ export abstract class RemoteHandlerAbstract {
   abstract outputFile(file: string, data: string | Buffer, options?: OutputFileOptions): Promise<void>
   abstract writeFile(file: string, data: string | Buffer, options?: WriteFileOptions): Promise<void>
 
-  abstract write(file: FileArg, buffer: Buffer, position?: number): Promise<void>
+  abstract write(file: FileDescriptor, buffer: Buffer, position?: number): Promise<void>
   abstract truncate(file: string, len: number): Promise<void>
 
   // ── File management ──────────────────────────────────────────────────────
@@ -156,14 +165,14 @@ export abstract class RemoteHandlerAbstract {
   abstract closeFile(fd: FileDescriptor): Promise<void>
 
   abstract getInfo(): Promise<RemoteInfoResult>
-
+  abstract getLinkCount(path: string): Promise<number>
   abstract getSize(file: FileDescriptor): Promise<number>
   abstract getSizeOnDisk(file: FileDescriptor): Promise<number>
   abstract lock(path: string): Promise<LockDisposer>
   abstract test(): Promise<TestResult>
   abstract isImmutable(): boolean
-  abstract getConfig(key: 'useVhdDirectory'): boolean | undefined
   abstract getConfig(key: 'compressionType'): string | undefined
+  abstract getConfig(key: 'diskFormat'): DiskFormat
   abstract getConfig(key: string): unknown
   abstract addPrefix(prefix: string): RemoteHandlerAbstract
 }

@xen-orchestra/fs/src/types/path.mts

@@ -1,6 +1,7 @@
 export { basename, dirname, join } from 'path'
 
 export declare const normalize: (path: string) => string
+export declare function isInDir(path: string, dir: string): boolean
 export declare function split(path: string): string[]
 export declare const relativeFromFile: (file: string, path: string) => string
 export declare const resolveFromFile: (file: string, path: string) => string

@xen-orchestra/qa-test/.USAGE.md

@@ -53,6 +53,13 @@ The following are required only for mirror backup tests (`qa:mirror`):
 | `MIRROR_DESTINATION_REPOSITORY_NAME` | Mirror destination repository name                                                                       | `Test mirror QA`             |
 | `MIRROR_DESTINATION_REPOSITORY_URL`  | `@xen-orchestra/fs` URL for the mirror destination. Same backend flexibility as `BACKUP_REPOSITORY_URL`. | `file:///tmp/xo-test-mirror` |
 
+The following are optional, for the hbd (deduplicated disks) tests (`qa:backup:hbd`). The suite is skipped when they are not set:
+
+| Variable                     | Description                                                                                                             | Example                                    |
+| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ |
+| `HBD_BACKUP_REPOSITORY_NAME` | hbd backup repository name                                                                                              | `Test hbd QA`                              |
+| `HBD_BACKUP_REPOSITORY_URL`  | `@xen-orchestra/fs` URL with `diskFormat="hbd"`. Needs hard links: not on S3 / Azure. Disk files are checked when local | `file:///tmp/xo-test-hbd?diskFormat="hbd"` |
+
 The following are optional, for load tests (disk churn via SSH). Leave `TEST_VM_SSH_KEY` unset
 to let `ssh` fall back to its own default identity files / `ssh-agent`:
 
@@ -87,6 +94,7 @@ Log namespaces follow the pattern `qa:<suite>[:<variant>]`:
 | `qa:backup:nbd`        | `tests/backup.nbd.test.js`                  |
 | `qa:backup:combined`   | `tests/backup-replication-combined.test.js` |
 | `qa:mirror`            | `tests/backup-mirror.test.js`               |
+| `qa:backup:hbd`        | `tests/backup.hbd.test.js`                  |
 | `qa:export`            | `tests/export.vhd.test.js`                  |
 | `qa:load:backup:delta` | `scripts/backup-load-delta.mjs`             |
 
@@ -106,6 +114,7 @@ yarn workspace @xen-orchestra/qa-test qa:vm                # VM lifecycle (clone
 yarn workspace @xen-orchestra/qa-test qa:backup            # Full and delta backup
 yarn workspace @xen-orchestra/qa-test qa:backup:nbd        # Backup with NBD protocol
 yarn workspace @xen-orchestra/qa-test qa:backup:combined   # Advanced scenarios (replication + backup)
+yarn workspace @xen-orchestra/qa-test qa:backup:hbd        # Backup, restore and mirror on a hbd repository
 yarn workspace @xen-orchestra/qa-test qa:export            # VHD/XVA export and restoration
 ```
 

@xen-orchestra/qa-test/.env.example

@@ -31,6 +31,15 @@ BACKUP_REPOSITORY_URL="file:///tmp/xo-test-backups"
 MIRROR_DESTINATION_REPOSITORY_NAME=Test mirror QA
 MIRROR_DESTINATION_REPOSITORY_URL="file:///tmp/xo-test-mirror"
 
+# ===========================================
+# HBD (DEDUPLICATED DISKS) TEST CONFIGURATION
+# ===========================================
+
+# Optional: the hbd tests are skipped when these are not set
+# The URL must set diskFormat="hbd"; hard links are required (not S3 / Azure)
+HBD_BACKUP_REPOSITORY_NAME=Test hbd QA
+HBD_BACKUP_REPOSITORY_URL='file:///tmp/xo-test-hbd?diskFormat="hbd"'
+
 # ===========================================
 # REPLICATION TEST CONFIGURATION
 # ===========================================

@xen-orchestra/qa-test/README.md

@@ -61,6 +61,13 @@ The following are required only for mirror backup tests (`qa:mirror`):
 | `MIRROR_DESTINATION_REPOSITORY_NAME` | Mirror destination repository name                                                                       | `Test mirror QA`             |
 | `MIRROR_DESTINATION_REPOSITORY_URL`  | `@xen-orchestra/fs` URL for the mirror destination. Same backend flexibility as `BACKUP_REPOSITORY_URL`. | `file:///tmp/xo-test-mirror` |
 
+The following are optional, for the hbd (deduplicated disks) tests (`qa:backup:hbd`). The suite is skipped when they are not set:
+
+| Variable                     | Description                                                                                                             | Example                                    |
+| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ |
+| `HBD_BACKUP_REPOSITORY_NAME` | hbd backup repository name                                                                                              | `Test hbd QA`                              |
+| `HBD_BACKUP_REPOSITORY_URL`  | `@xen-orchestra/fs` URL with `diskFormat="hbd"`. Needs hard links: not on S3 / Azure. Disk files are checked when local | `file:///tmp/xo-test-hbd?diskFormat="hbd"` |
+
 The following are optional, for load tests (disk churn via SSH). Leave `TEST_VM_SSH_KEY` unset
 to let `ssh` fall back to its own default identity files / `ssh-agent`:
 
@@ -95,6 +102,7 @@ Log namespaces follow the pattern `qa:<suite>[:<variant>]`:
 | `qa:backup:nbd`        | `tests/backup.nbd.test.js`                  |
 | `qa:backup:combined`   | `tests/backup-replication-combined.test.js` |
 | `qa:mirror`            | `tests/backup-mirror.test.js`               |
+| `qa:backup:hbd`        | `tests/backup.hbd.test.js`                  |
 | `qa:export`            | `tests/export.vhd.test.js`                  |
 | `qa:load:backup:delta` | `scripts/backup-load-delta.mjs`             |
 
@@ -114,6 +122,7 @@ yarn workspace @xen-orchestra/qa-test qa:vm                # VM lifecycle (clone
 yarn workspace @xen-orchestra/qa-test qa:backup            # Full and delta backup
 yarn workspace @xen-orchestra/qa-test qa:backup:nbd        # Backup with NBD protocol
 yarn workspace @xen-orchestra/qa-test qa:backup:combined   # Advanced scenarios (replication + backup)
+yarn workspace @xen-orchestra/qa-test qa:backup:hbd        # Backup, restore and mirror on a hbd repository
 yarn workspace @xen-orchestra/qa-test qa:export            # VHD/XVA export and restoration
 ```
 

@xen-orchestra/qa-test/package.json

@@ -35,6 +35,7 @@
     "qa:vm": "node --env-file-if-exists=.env --test tests/vm.test.js",
     "qa:backup:base": "node --env-file-if-exists=.env --test tests/backup.test.js",
     "qa:backup:nbd": "node --env-file-if-exists=.env --test tests/backup.nbd.test.js",
+    "qa:backup:hbd": "node --env-file-if-exists=.env --test --test-concurrency=1 tests/backup.hbd.test.js",
     "qa:backup:combined": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js",
     "qa:backup:combined:cycle": "node --env-file-if-exists=.env --test tests/backup-replication-combined.test.js --test-name-pattern='Full Cycle'",
     "qa:backup:sync": "node --env-file-if-exists=.env --test tests/backup.synchronized.test.js --test-concurrency=1",

@xen-orchestra/qa-test/tests/backup.hbd.test.js

@@ -0,0 +1,269 @@
+import assert from 'node:assert'
+import { after, before, describe, it } from 'node:test'
+import { createLogger } from '@xen-orchestra/log'
+import { getHandler, getSyncedHandler } from '@xen-orchestra/fs'
+
+import { backupConfig } from '../backup.config.js'
+import {
+  assertFullOrDelta,
+  assertHealthCheckExists,
+  assertHealthCheckSuccess,
+  generateBackupJobName,
+  getDefaultSchedule,
+  getRequiredEnv,
+  getScheduleKey,
+} from '../utils/index.js'
+import { assertBackupSuccess, assertRepositoryEmpty, resolveOrCreateBackupRepository } from '../utils/backupUtils.js'
+import { setup, teardown } from './setup.js'
+
+const log = createLogger('qa:backup:hbd')
+
+// optional: the whole suite is skipped when no hbd repository is configured
+const HBD_REPOSITORY_NAME = process.env.HBD_BACKUP_REPOSITORY_NAME
+const HBD_REPOSITORY_URL = process.env.HBD_BACKUP_REPOSITORY_URL
+const skip =
+  HBD_REPOSITORY_NAME && HBD_REPOSITORY_URL
+    ? false
+    : 'HBD_BACKUP_REPOSITORY_NAME and HBD_BACKUP_REPOSITORY_URL are not set'
+
+const RETENTION = 2
+
+/**
+ * Every file below `dir`, relative to the remote root
+ * @param {import('@xen-orchestra/fs').RemoteHandlerAbstract} handler
+ * @param {string} dir
+ * @returns {Promise<string[]>}
+ */
+async function listFiles(handler, dir) {
+  const found = []
+  for (const entry of await handler.list(dir, { prependDir: true, ignoreMissing: true })) {
+    try {
+      found.push(...(await listFiles(handler, entry)))
+    } catch (error) {
+      if (error.code !== 'ENOTDIR') {
+        throw error
+      }
+      found.push(entry)
+    }
+  }
+  return found
+}
+
+describe('Backup on a hbd (deduplicated) repository', { skip }, () => {
+  /** @type {import('../client/dispatchClient.js').DispatchClient} */
+  let dispatchClient
+  /** @type {import('../utils/resourceTracker.js').ResourceTracker} */
+  let tracker
+  let vm
+  /** the regular (VHD) repository from the .env, used as mirror source */
+  let vhdRepository
+  let hbdRepository
+  let healthCheckSrId
+  /** Set to true only after assertRepositoryEmpty passes: guards after() from purging data it did not create */
+  let reposVerifiedEmpty = false
+
+  const countVmBackups = async repositoryId => {
+    const backups = await dispatchClient.backup.listVmBackups([repositoryId])
+    return backups[repositoryId]?.[vm.uuid]?.length ?? 0
+  }
+
+  const listVmBackups = async repositoryId => {
+    const backups = await dispatchClient.backup.listVmBackups([repositoryId])
+    return [...(backups[repositoryId]?.[vm.uuid] ?? [])].sort((a, b) => a.timestamp - b.timestamp)
+  }
+
+  const purgeBackupData = async (...repositories) => {
+    for (const repository of repositories) {
+      if (repository === undefined) {
+        continue
+      }
+      try {
+        const ids = (await listVmBackups(repository.id)).map(backup => backup.id)
+        if (ids.length > 0) {
+          await dispatchClient.backup.deleteVmBackups(ids)
+        }
+      } catch (error) {
+        log.warn('Backup purge failed', { repository: repository.name, error })
+      }
+    }
+  }
+
+  /** Creates a backup job (tracked for cleanup), returns a function running it once */
+  const createBackupJob = async (repository, mode, options = {}) => {
+    const name = generateBackupJobName()
+    const config = backupConfig(name, getDefaultSchedule(), vm, repository, options)
+    config.mode = mode
+    config.settings[''].exportRetention = RETENTION
+
+    const jobId = await dispatchClient.backup.createBackupJob(config)
+    tracker.trackResource('backupJob', jobId, { name, mode })
+    const scheduleKey = getScheduleKey(await dispatchClient.backup.details(jobId))
+    assert.ok(scheduleKey, 'Backup schedule key is required')
+    tracker.trackResource('schedule', scheduleKey, { name, backupJobId: jobId })
+
+    return async context => {
+      const result = await dispatchClient.backup.runJobAndGetLog(jobId, scheduleKey)
+      assertBackupSuccess(result, context)
+      return result
+    }
+  }
+
+  /** Creates a mirror job (tracked for cleanup), returns a function running it once */
+  const createMirrorJob = async (source, destination, mode) => {
+    const name = generateBackupJobName()
+    const jobId = await dispatchClient.backup.createMirrorBackupJob({
+      name,
+      mode,
+      sourceRemote: source.id,
+      remotes: { [destination.id]: {} },
+      schedules: { '': getDefaultSchedule() },
+      settings: { '': { exportRetention: RETENTION, mergeBackupsSynchronously: true, bypassVdiChainsCheck: true } },
+    })
+    tracker.trackResource('backupJob', jobId, { name, mode, type: 'mirrorBackup' })
+    const scheduleKey = getScheduleKey(await dispatchClient.backup.details(jobId))
+    assert.ok(scheduleKey, 'Mirror schedule key is required')
+    tracker.trackResource('schedule', scheduleKey, { name, backupJobId: jobId })
+
+    return async context => {
+      const result = await dispatchClient.backup.runMirrorJobAndGetLog(jobId, scheduleKey)
+      assertBackupSuccess(result, context)
+      return result
+    }
+  }
+
+  /**
+   * Disk files of the VM on the hbd repository, only when it is reachable from here (file://)
+   * @returns {Promise<string[] | undefined>}
+   */
+  const listHbdDiskFiles = async () => {
+    if (!HBD_REPOSITORY_URL.startsWith('file://')) {
+      return undefined
+    }
+    const { value: handler, dispose } = await getSyncedHandler({ url: HBD_REPOSITORY_URL })
+    try {
+      return (await listFiles(handler, `xo-vm-backups/${vm.uuid}/vdis`)).filter(
+        path => path.endsWith('.hbd') || path.endsWith('.vhd')
+      )
+    } finally {
+      await dispose()
+    }
+  }
+
+  before(async () => {
+    let vms
+    ;({ dispatchClient, tracker, vms, backupRepository: vhdRepository } = await setup())
+    vm = vms[0]
+    assert.ok(vm, 'Setup should provide a test VM')
+
+    // fail fast on a URL that would silently write VHDs
+    assert.strictEqual(
+      getHandler({ url: HBD_REPOSITORY_URL }).getConfig('diskFormat'),
+      'hbd',
+      `HBD_BACKUP_REPOSITORY_URL must set diskFormat="hbd" (got ${HBD_REPOSITORY_URL})`
+    )
+
+    hbdRepository = await resolveOrCreateBackupRepository(dispatchClient, tracker, {
+      name: HBD_REPOSITORY_NAME,
+      url: HBD_REPOSITORY_URL,
+    })
+    assert.notStrictEqual(hbdRepository.id, vhdRepository.id, 'the hbd and VHD repositories must be distinct')
+
+    healthCheckSrId = (await dispatchClient.sr.details(getRequiredEnv('SR_ID'))).uuid
+
+    await assertRepositoryEmpty(dispatchClient, vhdRepository)
+    await assertRepositoryEmpty(dispatchClient, hbdRepository)
+    reposVerifiedEmpty = true
+  })
+
+  after(async () => {
+    if (reposVerifiedEmpty) {
+      await purgeBackupData(vhdRepository, hbdRepository)
+    }
+    if (dispatchClient && tracker) {
+      await teardown(dispatchClient, tracker)
+    }
+  })
+
+  describe('Incremental backup', () => {
+    it('chains deltas on .hbd disks and merges them past retention, with health checks', async () => {
+      const runBackup = await createBackupJob(hbdRepository, 'delta', { healthCheckSr: healthCheckSrId })
+
+      // run 1: full, run 2: delta, runs 3 and 4: delta + merge of the oldest backup
+      const RUNS = RETENTION + 2
+      for (let index = 0; index < RUNS; index++) {
+        const result = await runBackup(`hbd delta backup ${index + 1}/${RUNS}`)
+        // a full after the first run means the writer did not find its .hbd parent
+        assertFullOrDelta(result, hbdRepository.id, { mustBeFull: index === 0 })
+        // the health check restores the backup on a VM and boots it
+        assertHealthCheckExists(result)
+        assertHealthCheckSuccess(result)
+      }
+
+      const backups = await listVmBackups(hbdRepository.id)
+      assert.strictEqual(backups.length, RETENTION, `retention ${RETENTION} keeps ${RETENTION} backups`)
+
+      const diskFiles = await listHbdDiskFiles()
+      if (diskFiles !== undefined) {
+        log.debug('Disk files on the hbd repository', { diskFiles })
+        assert.ok(diskFiles.length > 0, 'the backups have disk files')
+        assert.deepStrictEqual(
+          diskFiles.filter(path => !path.endsWith('.hbd')),
+          [],
+          'every disk is written as .hbd'
+        )
+        // merged chains: one disk per kept backup and per VDI, nothing left by the merges
+        assert.strictEqual(diskFiles.length % RETENTION, 0, `disk files: ${diskFiles.join(', ')}`)
+      }
+    })
+
+    it('restores the latest delta as a new VM', async () => {
+      const backups = await listVmBackups(hbdRepository.id)
+      assert.ok(backups.length > 0, 'requires the backups of the previous test')
+      const latest = backups[backups.length - 1]
+      assert.strictEqual(latest.mode, 'delta')
+
+      const restoredVmUuid = await dispatchClient.backup.importVmBackup(latest.id, healthCheckSrId)
+      assert.ok(restoredVmUuid, 'the restore returns the new VM')
+      tracker.trackResource('vm', restoredVmUuid, { name: 'restored from hbd' })
+
+      const restoredVm = await dispatchClient.vm.details(restoredVmUuid)
+      assert.ok(restoredVm, 'the restored VM exists')
+      assert.strictEqual(
+        (await dispatchClient.vdi.getVdisForVm(restoredVmUuid)).length,
+        (await dispatchClient.vdi.getVdisForVm(vm.uuid)).length,
+        'the restored VM has as many disks as the source'
+      )
+    })
+
+    // file restore mounts the disks with VHD-only tools (@vates/fuse-vhd, vhdimount)
+    it.todo('restores files from a hbd backup')
+  })
+
+  describe('Mirror', () => {
+    before(async () => {
+      await purgeBackupData(vhdRepository, hbdRepository)
+    })
+
+    it('mirrors incremental backups from a VHD repository into the hbd one', async () => {
+      const runBackup = await createBackupJob(vhdRepository, 'delta')
+      const runMirror = await createMirrorJob(vhdRepository, hbdRepository, 'delta')
+
+      await runBackup('VHD delta backup 1/2')
+      await runMirror('mirror VHD -> hbd 1/2')
+      assert.ok((await countVmBackups(hbdRepository.id)) > 0, 'the hbd repository holds the mirrored backup')
+
+      // the second one is chained on the .hbd mirrored by the first one
+      await runBackup('VHD delta backup 2/2')
+      await runMirror('mirror VHD -> hbd 2/2')
+
+      const diskFiles = await listHbdDiskFiles()
+      if (diskFiles !== undefined) {
+        assert.deepStrictEqual(
+          diskFiles.filter(path => !path.endsWith('.hbd')),
+          [],
+          'the mirror writes .hbd disks, whatever the source format'
+        )
+      }
+    })
+  })
+})