feat(rest-api): export vdi and vdi snapshot content as qcow2 + vdi export fixes (#10350)

@vates/types/package.json

@@ -33,7 +33,8 @@
     "@eslint/js": "^9.19.0",
     "typescript": "~5.6.3",
     "typescript-eslint": "^8.61.0",
-    "@types/express": "^5.0.0"
+    "@types/express": "^5.0.0",
+    "@xen-orchestra/disk-transform": "^1.3.3"
   },
   "scripts": {
     "build": "tsc",

@vates/types/src/lib/xen-orchestra-xapi.mts

@@ -24,6 +24,7 @@ import type {
   VIF_LOCKING_MODE,
   VM_OPERATIONS,
 } from '../common.mjs'
+import type { DiskPassthrough } from '@xen-orchestra/disk-transform'
 import type { PassThrough, Readable } from 'node:stream'
 import type {
   XapiXoRecord,
@@ -476,3 +477,42 @@ export interface Xapi {
     }
   }
 }
+
+/**
+ * Parameters accepted by the `XapiDiskSource` constructor.
+ *
+ * `baseRef` turns the export into a differencing one: leaving it out exports the whole disk.
+ * `preferNbd` only expresses a preference, {@link XapiDiskSource} silently falls back to a plain
+ * stream export — and, when `baseRef` can't be used as a base, to a full one.
+ */
+export interface XapiDiskSourceOptions {
+  baseRef?: XenApiVdi['$ref']
+  blockSize?: number
+  nbdConcurrency?: number
+  /** When true, only `getBlockIndexes()` may be called: `readBlock()` throws. */
+  onlyListChangedBlocks?: boolean
+  preferNbd?: boolean
+  timeout?: number
+  vdiRef: XenApiVdi['$ref']
+  xapi: Xapi
+}
+
+/**
+ * Disk source handling the fallback logic of a VDI export: NBD + CBT, then NBD + stream export
+ * for the block list, then plain stream export.
+ *
+ * {@link XapiDiskSourceOptions.preferNbd} is a preference, not a guarantee: call
+ * {@link XapiDiskSource.useNbd} and {@link XapiDiskSource.useCbt} after `init()` to know what the
+ * export actually used.
+ */
+export interface XapiDiskSource extends DiskPassthrough {
+  useCbt(): boolean
+  useNbd(): boolean
+}
+
+/**
+ * Signature of `@xen-orchestra/xapi`'s `parseDateTime`.
+ *
+ * Returns a Unix timestamp in seconds, or `null` if the field is empty (as encoded by XAPI).
+ */
+export type ParseDateTime = (input: string | number | Date) => number | null

@vates/types/tsconfig.json

@@ -4,6 +4,7 @@
     "declaration": true,
     "strict": true,
     "module": "ESNext",
+    "moduleResolution": "bundler",
     "target": "ES2022",
     "skipLibCheck": true,
     "types": ["node"] // Only use types from Node.js and packages that are explicitly imported

@xen-orchestra/qcow2/src/consumer/ConsumerQcowStream.mts

@@ -23,7 +23,7 @@ function getAlignedBuffer(length: number): Buffer {
 /**
  * Extended Readable stream type that may include a length property
  */
-type WithLength<T> = T & { length?: number }
+export type WithLength<T> = T & { length?: number }
 
 /**
  * Generates a valid QCOW2 stream from a Disk.
@@ -373,7 +373,7 @@ export class QcowStreamGenerator {
  * @param disk The disk to convert
  * @param options Optional options
  * @param options.signal Optional AbortSignal to cancel the stream
- * @returns Readable stream of QCOW2 data
+ * @returns Readable stream of QCOW2 data, `length` is the exact size of the generated file
  */
 export async function toQcow2Stream(
   disk: Disk,

@xen-orchestra/qcow2/src/index.mts

@@ -1,3 +1,3 @@
 export { QcowStream } from './disk/QcowStream.mjs'
 export { QCowAccessor } from './disk/QcowAccessor.mjs'
-export { QcowStreamGenerator, toQcow2Stream } from './consumer/ConsumerQcowStream.mjs'
+export { QcowStreamGenerator, toQcow2Stream, type WithLength } from './consumer/ConsumerQcowStream.mjs'

@xen-orchestra/rest-api/package.json

@@ -28,6 +28,7 @@
     "@eslint/js": "^9.19.0",
     "@types/express": "^5.0.0",
     "@types/swagger-ui-express": "^4.1.7",
+    "@xen-orchestra/disk-transform": "^1.3.3",
     "openapi-types": "^12.1.3",
     "rimraf": "^6.0.1",
     "typescript": "~5.6.3",
@@ -40,6 +41,7 @@
     "@xen-orchestra/acl": "^1.6.1",
     "@xen-orchestra/backups": "^0.74.2",
     "@xen-orchestra/log": "^0.7.2",
+    "@xen-orchestra/qcow2": "^1.3.1",
     "@xen-orchestra/xapi": "^8.10.5",
     "complex-matcher": "^1.1.1",
     "golike-defer": "^0.5.1",
@@ -51,6 +53,7 @@
     "swagger-ui-express": "^5.0.1",
     "tsoa": "^6.6.0",
     "value-matcher": "^0.2.0",
+    "vhd-lib": "^4.16.1",
     "xo-common": "^0.11.0",
     "xo-remote-parser": "^0.10.1",
     "zod": "^4.3.6",

@xen-orchestra/rest-api/src/lib-types/vhd-lib.d.ts

@@ -0,0 +1,27 @@
+// `vhd-lib` is plain JS without published typings: this only binds the names used by the REST API
+//
+// @todo move these types to `@vates/types` so `@xen-orchestra/disk-cli`, which declares the same
+// module on its own, shares them
+declare module 'vhd-lib/disk-consumer/index.mjs' {
+  import type { Disk } from '@xen-orchestra/disk-transform'
+  import type { Readable } from 'node:stream'
+
+  /**
+   * `parentUuid` and `parentPath` describe the parent of a differencing VHD and must be given
+   * together, `uuid` identifies the produced VHD itself.
+   */
+  export interface ToVhdStreamOptions {
+    parentPath?: string
+    parentUuid?: Buffer
+    signal?: AbortSignal
+    uuid?: Buffer
+  }
+
+  /**
+   * The `length` of the returned stream is the exact size of the resulting VHD file.
+   *
+   * It is set by `DiskConsumerVhdStream` once the geometry is known, before the first byte is
+   * emitted, so it can be used as a `Content-Length`.
+   */
+  export function toVhdStream(disk: Disk, options?: ToVhdStreamOptions): Promise<Readable & { length: number }>
+}

@xen-orchestra/rest-api/src/lib-types/xen-orchestra-xapi.d.ts

@@ -0,0 +1,17 @@
+// `@xen-orchestra/xapi` is plain JS without published typings: this only binds the names used by
+// the REST API to the shapes declared in `@vates/types`
+declare module '@xen-orchestra/xapi' {
+  import type { ParseDateTime, XapiDiskSource, XapiDiskSourceOptions } from '@vates/types'
+
+  /**
+   * Biggest disk size that can be represented in a VHD file.
+   *
+   * Note: nothing in `vhd-lib` rejects a bigger disk, the geometry is silently
+   * clamped instead, therefore callers must check this limit themselves.
+   */
+  export const VHD_MAX_SIZE: number
+
+  export const parseDateTime: ParseDateTime
+
+  export const XapiDiskSource: new (params: XapiDiskSourceOptions) => XapiDiskSource
+}

@xen-orchestra/rest-api/src/vdi-snapshots/vdi-snapshot.controller.mts

@@ -17,7 +17,7 @@ import {
 import { inject } from 'inversify'
 import { provide } from 'inversify-binding-decorators'
 import type { Readable } from 'node:stream'
-import type { Request as ExRequest, Response as ExResponse } from 'express'
+import type { Request as ExRequest } from 'express'
 import type { SUPPORTED_VDI_FORMAT, XoAlarm, XoMessage, XoTask, XoVdiSnapshot } from '@vates/types'
 
 import { acl } from '../middlewares/acl.middleware.mjs'
@@ -104,13 +104,17 @@ export class VdiSnapshotController extends XapiXoController<XoVdiSnapshot> {
   async exportVdiSnapshotContent(
     @Request() req: ExRequest,
     @Path() id: string,
-    @Path() format: Exclude<SUPPORTED_VDI_FORMAT, 'qcow2'>
+    @Path() format: SUPPORTED_VDI_FORMAT
   ): Promise<Readable> {
-    const res = req.res as ExResponse
-    const stream = await this.#vdiService.exportContent(id as XoVdiSnapshot['id'], 'VDI-snapshot', {
-      format,
-      response: res,
-    })
+    const stream = await this.#vdiService.exportContent(id as XoVdiSnapshot['id'], 'VDI-snapshot', { format })
+
+    this.setHeader('content-disposition', `attachment; filename=${id}.${format}`)
+    this.setHeader('content-type', 'application/octet-stream')
+    if (stream.length !== undefined) {
+      // the size of an export is always known in advance, whatever its format
+      this.setHeader('content-length', String(stream.length))
+    }
+
     process.on('SIGTERM', () => req.destroy())
     req.on('close', () => stream.destroy())
     return stream

@xen-orchestra/rest-api/src/vdis/vdi.controller.mts

@@ -20,7 +20,7 @@ import {
 import { inject } from 'inversify'
 import { provide } from 'inversify-binding-decorators'
 import type { Readable } from 'node:stream'
-import { json, type Request as ExRequest, type Response as ExResponse } from 'express'
+import { json, type Request as ExRequest } from 'express'
 import type { SUPPORTED_VDI_FORMAT, Xapi, XoAlarm, XoMessage, XoSr, XoTask, XoVdi } from '@vates/types'
 
 import { SUPPORTED_ACTIONS_BY_RESOURCE, type SupportedActions } from '@xen-orchestra/acl'
@@ -133,10 +133,17 @@ export class VdiController extends XapiXoController<XoVdi> {
   async exportVdiContent(
     @Request() req: ExRequest,
     @Path() id: string,
-    @Path() format: Exclude<SUPPORTED_VDI_FORMAT, 'qcow2'>
+    @Path() format: SUPPORTED_VDI_FORMAT
   ): Promise<Readable> {
-    const res = req.res as ExResponse
-    const stream = await this.#vdiService.exportContent(id as XoVdi['id'], 'VDI', { format, response: res })
+    const stream = await this.#vdiService.exportContent(id as XoVdi['id'], 'VDI', { format })
+
+    this.setHeader('content-disposition', `attachment; filename=${id}.${format}`)
+    this.setHeader('content-type', 'application/octet-stream')
+    if (stream.length !== undefined) {
+      // the size of an export is always known in advance, whatever its format
+      this.setHeader('content-length', String(stream.length))
+    }
+
     process.on('SIGTERM', () => req.destroy())
     req.on('close', () => stream.destroy())
     return stream

@xen-orchestra/rest-api/src/vdis/vdi.service.mts

@@ -1,7 +1,19 @@
-import type { SUPPORTED_VDI_FORMAT, XoVdi, XoVdiSnapshot } from '@vates/types'
+import { createLogger } from '@xen-orchestra/log'
+import { toQcow2Stream } from '@xen-orchestra/qcow2'
+import { SUPPORTED_VDI_FORMAT } from '@vates/types'
+import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
+import { VHD_MAX_SIZE, XapiDiskSource } from '@xen-orchestra/xapi'
+import type { Disk } from '@xen-orchestra/disk-transform'
+import type { XoVdi, XoVdiSnapshot } from '@vates/types'
 import type { Readable } from 'node:stream'
+
+import { ApiError } from '../helpers/error.helper.mjs'
+import type { MaybePromise } from '../helpers/helper.type.mjs'
 import type { RestApi } from '../rest-api/rest-api.mjs'
-import type { Response as ExResponse } from 'express'
+
+const log = createLogger('xo:rest-api:vdi-service')
+
+type ExportedContentStream = Readable & { length?: number }
 
 export class VdiService {
   #restApi: RestApi
@@ -10,27 +22,80 @@ export class VdiService {
     this.#restApi = restApi
   }
 
-  async exportContent<Vdi extends XoVdi | XoVdiSnapshot>(
+  /**
+   * Open a `XapiDiskSource` on the VDI and let `toStream` rebuild the wanted format
+   * from its blocks, instead of relying on the export of the XAPI: this uses NBD when
+   * it is available, whatever the format the VDI is stored in, and gives the exact
+   * size of the export in advance.
+   */
+  async #exportContentFromDiskSource<Vdi extends XoVdi | XoVdiSnapshot>(
     id: Vdi['id'],
     type: Vdi['type'],
-    { format, response }: { format: SUPPORTED_VDI_FORMAT; response?: ExResponse }
-  ): Promise<Readable & { length?: number }> {
-    const xapiVdi = this.#restApi.getXapiObject<Vdi>(id, type)
-    const stream = await xapiVdi.$xapi.VDI_exportContent(xapiVdi.$ref, { format })
+    toStream: (disk: Disk) => MaybePromise<ExportedContentStream>
+  ): Promise<ExportedContentStream> {
+    const { $ref: vdiRef, $xapi: xapi } = this.#restApi.getXapiObject<Vdi>(id, type)
+    const disk = new XapiDiskSource({ xapi, vdiRef })
+    await disk.init()
 
-    if (response !== undefined) {
-      const headers = new Headers({
-        'content-disposition': `attachment; filename=${id}.${format}`,
-        'content-type': 'application/octet-stream',
-      })
+    const closeDisk = () =>
+      disk.close().catch(error => log.warn('failed to close the disk source', { error, vdiId: id }))
 
-      if (stream.length !== undefined) {
-        headers.append('content-length', stream.length.toString())
-      }
+    try {
+      const stream = await toStream(disk)
 
-      response.setHeaders(headers)
+      // the block generator of the disk closes it when it ends, but it is never
+      // started if the stream is destroyed before being consumed
+      stream.once('close', closeDisk)
+
+      return stream
+    } catch (error) {
+      await closeDisk()
+      throw error
     }
+  }
+
+  /**
+   * A raw export is a byte for byte copy of the disk: its size is the virtual size of
+   * the VDI, even though the XAPI does not announce it.
+   */
+  async #exportContentAsRaw<Vdi extends XoVdi | XoVdiSnapshot>(
+    id: Vdi['id'],
+    type: Vdi['type']
+  ): Promise<ExportedContentStream> {
+    // `virtual_size` is read from the XAPI record, which is fetched anyway to get the
+    // ref: no extra object lookup, and no `VDI.get_virtual_size` call
+    const { $ref: vdiRef, $xapi: xapi, virtual_size: size } = this.#restApi.getXapiObject<Vdi>(id, type)
+
+    const stream = await xapi.VDI_exportContent(vdiRef, { format: SUPPORTED_VDI_FORMAT.raw })
+    stream.length = size
 
     return stream
   }
+
+  /**
+   * `length` is set on the returned stream: the size of the export is always known in
+   * advance, it's up to the caller to expose it as a `content-length`.
+   */
+  async exportContent<Vdi extends XoVdi | XoVdiSnapshot>(
+    id: Vdi['id'],
+    type: Vdi['type'],
+    { format }: { format: SUPPORTED_VDI_FORMAT }
+  ): Promise<ExportedContentStream> {
+    if (format === SUPPORTED_VDI_FORMAT.vhd) {
+      const { size } = this.#restApi.getObject<Vdi>(id, type)
+      if (size > VHD_MAX_SIZE) {
+        throw new ApiError(`a VDI of ${size} bytes is too large to be exported as VHD`, 422, {
+          data: { maxSize: VHD_MAX_SIZE, size },
+        })
+      }
+
+      return this.#exportContentFromDiskSource<Vdi>(id, type, disk => toVhdStream(disk))
+    }
+
+    if (format === SUPPORTED_VDI_FORMAT.qcow2) {
+      return this.#exportContentFromDiskSource<Vdi>(id, type, disk => toQcow2Stream(disk))
+    }
+
+    return this.#exportContentAsRaw<Vdi>(id, type)
+  }
 }

@xen-orchestra/rest-api/src/vdis/vdi.service.test.mts

@@ -0,0 +1,146 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { Readable } from 'node:stream'
+import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
+import { SUPPORTED_VDI_FORMAT } from '@vates/types'
+import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
+import { VHD_MAX_SIZE } from '@xen-orchestra/xapi'
+
+import { ApiError } from '../helpers/error.helper.mjs'
+import { VdiService } from './vdi.service.mjs'
+import type { DiskBlock } from '@xen-orchestra/disk-transform'
+import type { RestApi } from '../rest-api/rest-api.mjs'
+import type { XoVdi } from '@vates/types'
+
+const VDI_ID = 'c77f9955-c1d2-4b39-aa1c-73cdb2dacb7e' as XoVdi['id']
+const VDI_REF = 'OpaqueRef:vdi-1'
+
+const BLOCK_SIZE = 2 * 1024 * 1024
+const NB_BLOCKS = 4
+const VIRTUAL_SIZE = NB_BLOCKS * BLOCK_SIZE
+// the disk is sparse: the formats that only store the allocated blocks must not announce
+// the virtual size
+const ALLOCATED_BLOCK_INDEXES = [0, 2]
+
+const EMPTY_BLOCK = Buffer.alloc(BLOCK_SIZE)
+const buildBlockData = (index: number) => Buffer.alloc(BLOCK_SIZE, index + 1)
+
+/**
+ * Source of the VHD export the XAPI is mocked to return: the exports rebuilt by XO are
+ * generated from the blocks of this disk.
+ */
+class InMemoryDisk extends RandomAccessDisk {
+  getVirtualSize(): number {
+    return VIRTUAL_SIZE
+  }
+  getBlockSize(): number {
+    return BLOCK_SIZE
+  }
+  getBlockIndexes(): number[] {
+    return [...ALLOCATED_BLOCK_INDEXES]
+  }
+  hasBlock(index: number): boolean {
+    return ALLOCATED_BLOCK_INDEXES.includes(index)
+  }
+  async readBlock(index: number): Promise<DiskBlock> {
+    return { index, data: buildBlockData(index) }
+  }
+  async init(): Promise<void> {}
+  async close(): Promise<void> {}
+  isDifferencing(): boolean {
+    return false
+  }
+}
+
+/** A raw export is byte for byte, unallocated blocks included. */
+function createRawExport(): Readable {
+  function* blocks() {
+    for (let index = 0; index < NB_BLOCKS; index++) {
+      yield ALLOCATED_BLOCK_INDEXES.includes(index) ? buildBlockData(index) : EMPTY_BLOCK
+    }
+  }
+
+  return Readable.from(blocks(), { objectMode: false })
+}
+
+/**
+ * The XAPI of this VDI only knows how to export it as raw or VHD: the other formats are
+ * rebuilt by XO from the blocks of a `XapiDiskSource`, which is fed by the VHD export
+ * here since NBD is not available.
+ */
+function createRestApi() {
+  const exportContentCalls: unknown[][] = []
+
+  const xapi = {
+    // the disk source reads the format the VDI is stored in from its `sm_config`
+    getField: async (_type: string, _ref: string, field: string) =>
+      field === 'sm_config' ? { 'image-format': SUPPORTED_VDI_FORMAT.vhd } : undefined,
+    VDI_exportContent: async (...args: unknown[]) => {
+      exportContentCalls.push(args)
+      const { format } = args[1] as { format: SUPPORTED_VDI_FORMAT }
+
+      return format === SUPPORTED_VDI_FORMAT.raw ? createRawExport() : toVhdStream(new InMemoryDisk())
+    },
+  }
+
+  const restApi = {
+    getObject: () => ({ id: VDI_ID, size: VIRTUAL_SIZE }),
+    getXapiObject: () => ({ $ref: VDI_REF, $xapi: xapi, virtual_size: VIRTUAL_SIZE }),
+  } as unknown as RestApi
+
+  return { exportContentCalls, restApi }
+}
+
+describe('VdiService.exportContent', () => {
+  for (const format of Object.values(SUPPORTED_VDI_FORMAT)) {
+    it(`announces, for the ${format} format, a length equal to the number of streamed bytes`, async () => {
+      const { restApi } = createRestApi()
+
+      const exported = await new VdiService(restApi).exportContent(VDI_ID, 'VDI', { format })
+
+      // `length` is only exposed to the controllers, which turn it into a content-length:
+      // it is sent before the export is generated, an export streaming more or less than
+      // that breaks the download
+      const announcedLength = exported.length
+      assert.ok(
+        Number.isInteger(announcedLength) && (announcedLength as number) > 0,
+        `${format} must announce an integer length, got ${announcedLength}`
+      )
+
+      let streamedBytes = 0
+      for await (const chunk of exported) {
+        streamedBytes += chunk.length
+      }
+
+      assert.strictEqual(streamedBytes, announcedLength)
+    })
+  }
+
+  it('exports the raw format through the XAPI, with the virtual size as length', async () => {
+    const { exportContentCalls, restApi } = createRestApi()
+
+    const exported = await new VdiService(restApi).exportContent(VDI_ID, 'VDI', { format: 'raw' })
+
+    // the XAPI does not announce the size of a raw export, XO computes it from the VDI
+    assert.strictEqual(exported.length, VIRTUAL_SIZE)
+    // a raw export is not rebuilt from the disk source: it is the export of the XAPI itself
+    assert.deepStrictEqual(exportContentCalls, [[VDI_REF, { format: 'raw' }]])
+
+    exported.destroy()
+  })
+
+  it('rejects a VHD export of a VDI larger than the VHD max size', async () => {
+    const size = VHD_MAX_SIZE + 1
+    const restApi = {
+      getObject: () => ({ id: VDI_ID, size }),
+      getXapiObject: () => assert.fail('the disk must not be opened when the size is not supported'),
+    } as unknown as RestApi
+
+    await assert.rejects(new VdiService(restApi).exportContent(VDI_ID, 'VDI', { format: 'vhd' }), (error: unknown) => {
+      assert(error instanceof ApiError)
+      assert.strictEqual(error.status, 422)
+      assert.deepStrictEqual(error.data, { maxSize: VHD_MAX_SIZE, size })
+      return true
+    })
+  })
+})

@xen-orchestra/rest-api/src/vms/vm.service.mts

@@ -303,6 +303,10 @@ export class VmService {
     for (const id in snapshotReplicas) {
       const snapshot = snapshotReplicas[id as XoVmSnapshot['id']]
       const timestamp = parseDateTime(snapshot.other['xo:backup:datetime'])
+      if (timestamp === null) {
+        // the snapshot has no backup date, it can't be the last replication
+        continue
+      }
 
       if (lastTimestamp === undefined || lastTimestamp < timestamp) {
         lastTimestamp = timestamp

CHANGELOG.unreleased.md

@@ -17,6 +17,7 @@
 - [Rolling pool update/reboot] A pool can now skip the phase which brings the VMs back to the host they were running on, which halves the migrations of the run (PR [#10295](https://github.com/vatesfr/xen-orchestra/pull/10295))
 - [XO5/Backups] Open the backup job edition form in the same tab when editing a backup job from the VM page (PR [#10342](https://github.com/vatesfr/xen-orchestra/pull/10342))
 - [Web-Core/TabItem] Update the component to remove uppercase for better readability (PR [#10338](https://github.com/vatesfr/xen-orchestra/pull/10338))
+- [REST API] VDI can now be exported in qcow2 format, and the VHD export uses NBD when available. Both formats work whatever the format the disk is stored in. Every export format, raw included, now provides the size of the download (PR [#10350](https://github.com/vatesfr/xen-orchestra/pull/10350))
 
 ### Bug fixes
 
@@ -50,10 +51,12 @@
 
 <!--packages-start-->
 
+- @vates/types minor
 - @xen-orchestra/backup-archive patch
 - @xen-orchestra/backups patch
 - @xen-orchestra/disk-cli patch
 - @xen-orchestra/qcow2 minor
+- @xen-orchestra/rest-api minor
 - @xen-orchestra/web minor
 - @xen-orchestra/web-core minor
 - @xen-orchestra/xapi patch