feat(qcow2): fix invalid range error (#9948)

Reduce memory consumption up to 700MB / TB of data ( 300-350Mb per call of getBlockIndexes().length , called twice).

Ticket: 59102
Tested in client environment. Works for a 7TB import

@xen-orchestra/disk-transform/src/Disk.mts

@@ -22,6 +22,9 @@ export abstract class Disk {
 
   abstract getVirtualSize(): number
   abstract getBlockSize(): number
+  getBlockIndexesCount(): number {
+    return this.getBlockIndexes().length
+  }
   abstract init(): Promise<void>
   abstract close(): Promise<void>
 

@xen-orchestra/disk-transform/src/DiskSmallerBlock.mts

@@ -72,6 +72,22 @@ export class DiskSmallerBlock extends DiskPassthrough {
   #isBlockBeforeEnd(blockIndex: number): boolean {
     return blockIndex < this.#maxBlockIndex
   }
+
+  getBlockIndexesCount(): number {
+    let count = this.source.getBlockIndexesCount() * this.#blockRatio
+
+    // Only the last source block can exceed #maxBlockIndex.
+    // If it is allocated, subtract the sub-blocks which are upper than the virtual size.
+    const lastSourceBlockIndex = Math.ceil(this.getVirtualSize() / this.source.getBlockSize()) - 1
+    if (this.source.hasBlock(lastSourceBlockIndex)) {
+      const rest = (lastSourceBlockIndex + 1) * this.#blockRatio - this.#maxBlockIndex
+      if (rest > 0) {
+        count -= rest
+      }
+    }
+    return count
+  }
+
   getBlockIndexes(): Array<number> {
     const blockRatio = this.source.getBlockSize() / this.getBlockSize()
     const sourceIndexes = this.source.getBlockIndexes()

@xen-orchestra/disk-transform/src/DiskSmallerBlock.test.mts

@@ -183,6 +183,51 @@ test('close propagation', async () => {
   assert(source.closed)
 })
 
+test('getBlockIndexesCount - last source block not allocated, no trimming needed', async () => {
+  // virtualSize=2048, sourceBlockSize=512, blockSize=256, blockRatio=2
+  // #maxBlockIndex=8, lastSourceBlockIndex=3, hasBlock(3)=false
+  // count = 2 * 2 = 4
+  const source = new MockDisk(512, 2048, [
+    [0, Buffer.alloc(512)],
+    [2, Buffer.alloc(512)],
+  ])
+  await source.init()
+  const disk = new DiskSmallerBlock(source, 256)
+  await disk.init()
+
+  assert.strictEqual(disk.getBlockIndexesCount(), 4)
+  assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
+})
+
+test('getBlockIndexesCount - last source block allocated and aligned, excess=0', async () => {
+  // virtualSize=1024, sourceBlockSize=512, blockSize=256, blockRatio=2
+  // #maxBlockIndex=4, lastSourceBlockIndex=1, hasBlock(1)=true
+  // excess = (1+1)*2 - 4 = 0 -> count = 1*2 = 2
+  const source = new MockDisk(512, 1024, [[1, Buffer.alloc(512)]])
+  await source.init()
+  const disk = new DiskSmallerBlock(source, 256)
+  await disk.init()
+
+  assert.strictEqual(disk.getBlockIndexesCount(), 2)
+  assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
+})
+
+test('getBlockIndexesCount - unaligned end, last source block allocated, excess>0', async () => {
+  // virtualSize=2304, sourceBlockSize=1024, blockSize=256, blockRatio=4
+  // #maxBlockIndex=9, lastSourceBlockIndex=2, hasBlock(2)=true
+  // excess = (2+1)*4 - 9 = 3 -> count = 2*4 - 3 = 5
+  const source = new MockDisk(1024, 2048 + 256, [
+    [0, Buffer.alloc(1024)],
+    [2, Buffer.alloc(1024)],
+  ])
+  await source.init()
+  const disk = new DiskSmallerBlock(source, 256)
+  await disk.init()
+
+  assert.strictEqual(disk.getBlockIndexesCount(), 5)
+  assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
+})
+
 test('unaligned END ', async () => {
   const source = new MockDisk(1024, 2048 + 256, [
     [0, Buffer.alloc(1024)],

@xen-orchestra/qcow2/src/consumer/ConsumerQcowStream.mts

@@ -31,6 +31,7 @@ type WithLength<T> = T & { length?: number }
 export class QcowStreamGenerator {
   #disk: Disk
   #offset = 0
+  #nbAllocatedBlocks = 0
 
   /**
    * Creates a new QCOW2 stream generator
@@ -48,6 +49,7 @@ export class QcowStreamGenerator {
       this.#disk = disk
     }
     assert.strictEqual(this.#disk.getBlockSize(), CLUSTER_SIZE)
+    this.#nbAllocatedBlocks = this.#disk.getBlockIndexesCount()
   }
 
   /**
@@ -104,7 +106,7 @@ export class QcowStreamGenerator {
    */
   #computeRefCountSize(addressTableSize: number): { refCountL1Size: number; refCountL2Size: number } {
     const disk = this.#disk
-    const nbBlocks = disk.getBlockIndexes().length
+    const nbBlocks = this.#nbAllocatedBlocks
 
     // Total clusters needed (header + addressing tables + data clusters)
     let nbAllocatedClusters = 1 /* header */ + addressTableSize / CLUSTER_SIZE + nbBlocks
@@ -241,9 +243,8 @@ export class QcowStreamGenerator {
    */
   stream(signal?: AbortSignal): WithLength<Readable> {
     const disk = this.#disk
-    const nbAllocatedBlocks = disk.getBlockIndexes().length
+    const nbAllocatedBlocks = this.#nbAllocatedBlocks
     const nbTotalBlock = Math.ceil(disk.getVirtualSize() / disk.getBlockSize())
-
     // Compute table sizes
     const { size: addressTableSize, nbL1Entries } = this.#computeAddressingSpace()
     const { refCountL1Size, refCountL2Size } = this.#computeRefCountSize(addressTableSize)

@xen-orchestra/qcow2/src/consumer/ConsumerQcowStream.test.mts

@@ -0,0 +1,76 @@
+import assert from 'node:assert'
+import { test } from 'node:test'
+import { DiskSmallerBlock, RandomAccessDisk, type DiskBlock } from '@xen-orchestra/disk-transform'
+import { toQcow2Stream } from './ConsumerQcowStream.mjs'
+
+
+class FullyAllocatedMockDisk extends RandomAccessDisk {
+  private blockSize: number
+  private virtualSize: number
+  private blocks: Map<number, Buffer>
+  public closed: boolean = false
+  public initialized: boolean = false
+  public isDiff: boolean = false
+  public parentDisk: RandomAccessDisk | null = null
+  private readonly nbBlocks: number
+
+  constructor(blockSize: number, virtualSize: number, blocks: [number, Buffer][] = []) {
+    super()
+    this.blockSize = blockSize
+    this.virtualSize = virtualSize
+    this.blocks = new Map(blocks)
+    this.nbBlocks = Math.ceil(virtualSize / blockSize)
+  }
+
+  async readBlock(index: number): Promise<DiskBlock> {
+    if (this.closed) throw new Error('Disk closed')
+    if (!this.initialized) throw new Error('Disk not initialized')
+    if (!this.blocks.has(index)) {
+      if (this.isDiff && this.parentDisk) {
+        return this.parentDisk.readBlock(index)
+      }
+      throw new Error(`Block ${index} not found`)
+    }
+    return { index, data: this.blocks.get(index)! }
+  }
+
+  getVirtualSize(): number {
+    return this.virtualSize
+  }
+  getBlockSize(): number {
+    return this.blockSize
+  }
+  async init(): Promise<void> {
+    this.initialized = true
+  }
+  async close(): Promise<void> {
+    this.closed = true
+  }
+  isDifferencing(): boolean {
+    return this.isDiff
+  }
+  instantiateParent(): RandomAccessDisk {
+    if (!this.parentDisk) throw new Error('No parent disk available')
+    return this.parentDisk
+  }
+
+  getBlockIndexes(): number[] {
+    return Array.from({ length: this.nbBlocks }, (_, i) => i)
+  }
+
+  hasBlock(index: number): boolean {
+    return index >= 0 && index < this.nbBlocks
+  }
+}
+
+// Regression test linked to ticket #59102
+test('does not throw RangeError: Invalid array length on large fully-allocated disk', async () => {
+  const VHD_BLOCK_SIZE = 2 * 1024 * 1024 // 2097152
+  const QCOW2_CLUSTER_SIZE = 64 * 1024 // 65536
+  const DISK_CAPACITY = 7696581394432 // 7.18 TB — exact value from bug report
+
+  const source = new FullyAllocatedMockDisk(VHD_BLOCK_SIZE, DISK_CAPACITY)
+  await source.init()
+  const disk = new DiskSmallerBlock(source, QCOW2_CLUSTER_SIZE)
+  assert.doesNotThrow(() => toQcow2Stream(disk))
+})

@xen-orchestra/qcow2/src/disk/QcowDisk.mts

@@ -52,7 +52,7 @@ export abstract class QcowDisk extends RandomAccessDisk {
   abstract readBuffer(offset: number, length: number): Promise<Buffer>
 
   async readBlock(index: number): Promise<DiskBlock> {
-    if (this.#nbClusterPerL2Table == 0 || this.#l2Increment == 0) {
+    if (this.#nbClusterPerL2Table === 0 || this.#l2Increment === 0) {
       throw new Error(`QcowDisk.init() has not been called`)
     }
 
@@ -152,8 +152,24 @@ export abstract class QcowDisk extends RandomAccessDisk {
     return blockIndexes
   }
 
+  getBlockIndexesCount(): number {
+    // an L2 table doesn't mean all its cluster slots are allocated
+    // so we must inspect each entry
+    let count = 0
+    this.#l2Tables.forEach(l2TableBuffer => {
+      if (l2TableBuffer) {
+        for (let i = 0; i < l2TableBuffer.length; i += this.#l2Increment) {
+          if (Number(l2TableBuffer.readBigUInt64BE(i) & 0x00ffffffffffe0n) !== 0) {
+            count++
+          }
+        }
+      }
+    })
+    return count
+  }
+
   hasBlock(index: number): boolean {
-    if (this.#nbClusterPerL2Table == 0 || this.#l2Increment == 0) {
+    if (this.#nbClusterPerL2Table === 0 || this.#l2Increment === 0) {
       throw new Error(`QcowDisk.init() has not been called`)
     }
 

CHANGELOG.unreleased.md

@@ -48,6 +48,7 @@
 - [Qcow2] Fix initialization Map range error for big (>1 TB) Qcow2 disks (PR [#9940](https://github.com/vatesfr/xen-orchestra/pull/9940))
 - **XO 5**:
   - [Jobs] fix array values being incorrectly handled (used for instance on job.runSequence) (PR [#9928](https://github.com/vatesfr/xen-orchestra/pull/9928))
+- [V2V] Fix stream issue for large disks used with smaller blocks (PR [#9948](https://github.com/vatesfr/xen-orchestra/pull/9948))
 
 - xo-server-sdn-controller: apply/clean network rules on VIF update (PR [#9933](https://github.com/vatesfr/xen-orchestra/pull/9933))
 - [Rest Api] Fix `possibly unhandled rejection invalid crendentials` (PR [#9938](https://github.com/vatesfr/xen-orchestra/pull/9938))
@@ -70,6 +71,7 @@
 
 - @vates/types minor
 - @xen-orchestra/acl minor
+- @xen-orchestra/disk-transform patch
 - @xen-orchestra/qcow2 patch
 - @xen-orchestra/rest-api minor
 - @xen-orchestra/xapi minor