Compare commits

..

4 Commits

Author SHA1 Message Date
Simon Pacotte
9b69b339a6 refix 2026-10-05 14:55:35 +02:00
Simon Pacotte
19b1fe1009 fix Florent 2026-10-05 10:53:45 +02:00
Simon Pacotte
86b9686c5c update changelog 2026-10-02 11:23:40 +02:00
Simon Pacotte
0eab596880 fix(backups): fix synchronized backup snapshop showing as vm in task log 2026-10-02 11:15:27 +02:00
1665 changed files with 14727 additions and 86379 deletions

View File

@@ -1,19 +1,8 @@
name: Vectura
# only when vectura changes: the rest of the repository does not build it
#
# pull requests rather than every push: GitHub filters a push on the diff with
# the previous push, which after a rebase on `master` (or a merge of it)
# contains the vectura changes of `master`, whereas it filters a pull request
# on the diff with the merge base, i.e. the changes of the branch only
on:
pull_request:
paths:
- '@xen-orchestra/vmware-explorer/vectura/**'
- .github/workflows/vectura.yml
push:
branches:
- master
paths:
- '@xen-orchestra/vmware-explorer/vectura/**'
- .github/workflows/vectura.yml
@@ -45,8 +34,8 @@ jobs:
- name: Hold the coverage floor
run: cargo test --locked --test coverage -- --ignored
# builds the binary as release.sh does, checks it, then checks the one in git,
# which xo-server runs, is that build
# builds the binary as a release does, then checks it and the one in git,
# which xo-server runs
binary:
runs-on: ubuntu-latest
steps:
@@ -65,9 +54,4 @@ jobs:
- run: scripts/build.sh "$RUNNER_TEMP/build"
- run: scripts/check-binary.sh "$RUNNER_TEMP/build/vectura"
- name: The binary in git is the build of these sources
run: |
cmp "$RUNNER_TEMP/build/vectura" vectura || {
echo '::error::vectura is not the build of these sources: run scripts/build.sh . in vectura/ and commit the binary'
exit 1
}
- run: scripts/check-binary.sh vectura

View File

@@ -19,12 +19,12 @@
},
"main": "./index.mjs",
"dependencies": {
"@xen-orchestra/fs": "^4.10.0",
"@xen-orchestra/fs": "^4.9.5",
"@xen-orchestra/log": "^0.7.2",
"fuse-native": "^2.2.6",
"lru-cache": "^7.14.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.17.0"
"vhd-lib": "^4.16.2"
},
"bin": {
"xo-fuse-vhd": "./cli.mjs"

View File

@@ -15,7 +15,7 @@ The target drives login negotiation onto a single, fixed code path:
It implements SendTargets discovery, login (`AuthMethod=None` or one-way
**CHAP** — see [Authentication](#authentication-chap)), the ~10 SCSI
commands a Linux initiator issues to attach and use a block device
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`/`0xb0`, `REPORT LUNS`, `READ CAPACITY 10/16`,
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`, `REPORT LUNS`, `READ CAPACITY 10/16`,
`READ`/`WRITE 10/16`, `TEST UNIT READY`, `REQUEST SENSE`, `MODE SENSE`,
`SYNCHRONIZE CACHE`), the full R2T / Data-Out write path, NOP keepalive replies,
and Logout. Commands may be outstanding concurrently (Data-Out PDUs are routed
@@ -28,19 +28,6 @@ The package also ships a userspace **initiator** (`IscsiInitiator`) and an
Not implemented (yet): mutual (two-way) CHAP, Task Management (abort/reset),
target-initiated NOP keepalives, multiple connections/LUNs.
What a peer can make the target do is bounded:
- connections that have not logged in yet are limited in time (`loginTimeoutMs`)
and number (`maxPendingConnections`), and `allowedInitiatorAddresses` (the
source addresses of the expected initiators) closes the other ones as soon as
they are accepted, reporting them to `onConnectionRefused`
- a PDU data segment is at most 8 KiB during login, then the declared
`MaxRecvDataSegmentLength` (256 KiB), checked before it is read
- a single `READ`/`WRITE` is at most `maxTransferLength` (8 MiB by default),
advertised in the block limits VPD page so that initiators split their I/O
- a `BlockDevice` with `readOnly: true` is reported write-protected, and `WRITE`
is refused with `DATA PROTECT` before any data is solicited
### CLI
The package ships a `vates-iscsi` binary that serves a file as a LUN:
@@ -152,8 +139,6 @@ class RandomAccessDiskLun {
}
return out
}
// reported write-protected: WRITE is refused before reaching write()
readOnly = true
write() {
throw new Error('read-only LUN')
}

View File

@@ -18,14 +18,14 @@ The package has two independent roles with very different maturity:
## Session and connection
| Aspect | Target | Initiator |
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------- |
| Connections per session | `MaxConnections=1`, enforced | opens exactly one |
| Session reinstatement | **no** — no ISID tracking; once a new connection has established a Normal session (CHAP included), it tears down and replaces whatever connection currently exists, rather than RFC 7143's ISID-matched reinstatement (see `index.mts`'s class doc for the reasoning: safe here because the target is ephemeral, single-consumer, and CHAP-guarded). Until then it does not affect the current one, and is bounded by `loginTimeoutMs` and `maxPendingConnections` | n/a (single connection) |
| Multiple LUNs | **no** — exactly one LUN, always LUN 0 | reads are hardcoded to LUN 0 |
| `ErrorRecoveryLevel` | `0` only | requests `0` |
| Header/data digests | `HeaderDigest=None`, `DataDigest=None` — no CRC32C on the wire | same |
| Markers (`IFMarker`/`OFMarker`) | not offered (RFC-deprecated) | not offered |
| Aspect | Target | Initiator |
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------- |
| Connections per session | `MaxConnections=1`, enforced | opens exactly one |
| Session reinstatement | **no** — no ISID tracking; a new connection always tears down and replaces whatever connection currently exists, rather than RFC 7143's ISID-matched reinstatement (see `index.mts`'s class doc for the reasoning: safe here because the target is ephemeral, single-consumer, and CHAP-guarded) | n/a (single connection) |
| Multiple LUNs | **no** — exactly one LUN, always LUN 0 | reads are hardcoded to LUN 0 |
| `ErrorRecoveryLevel` | `0` only | requests `0` |
| Header/data digests | `HeaderDigest=None`, `DataDigest=None` — no CRC32C on the wire | same |
| Markers (`IFMarker`/`OFMarker`) | not offered (RFC-deprecated) | not offered |
## Login / text negotiation
@@ -72,17 +72,17 @@ keys if it declines transit on the first exchange.
Decoded explicitly (`types.mts#decodeCdb`); anything else maps to `unsupported` → `CHECK CONDITION` /
`ILLEGAL_REQUEST` / `INVALID COMMAND OPERATION CODE`:
| Command | Variants |
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `INQUIRY` | standard + EVPD `0x00`/`0x80`/`0x83` (vendor/product/revision/serial from `identity`) and `0xb0` (block limits: `maxTransferLength`) |
| `REPORT LUNS` | |
| `READ CAPACITY` | `(10)` and `(16)` |
| `READ` | `(10)` and `(16)`; longer than `maxTransferLength` → `INVALID FIELD IN CDB` |
| `WRITE` | `(10)` and `(16)`; longer than `maxTransferLength` → `INVALID FIELD IN CDB`; on a `readOnly` LUN → `DATA PROTECT` / `WRITE PROTECTED`, without soliciting data |
| `TEST UNIT READY` | |
| `REQUEST SENSE` | |
| `MODE SENSE` | `(6)` and `(10)`; `WP` set on a `readOnly` LUN |
| `SYNCHRONIZE CACHE` | `(10)` and `(16)` — flushes the backing `BlockDevice` |
| Command | Variants |
| ------------------- | ---------------------------------------------------------------- |
| `INQUIRY` | standard + EVPD (vendor/product/revision/serial from `identity`) |
| `REPORT LUNS` | |
| `READ CAPACITY` | `(10)` and `(16)` |
| `READ` | `(10)` and `(16)` |
| `WRITE` | `(10)` and `(16)` |
| `TEST UNIT READY` | |
| `REQUEST SENSE` | |
| `MODE SENSE` | `(6)` and `(10)` |
| `SYNCHRONIZE CACHE` | `(10)` and `(16)` — flushes the backing `BlockDevice` |
**Not implemented**: persistent reservations, VAAI/thin-provisioning primitives (`UNMAP`, `WRITE SAME`, `COMPARE
AND WRITE`, ...), any vendor-specific commands — anything beyond the ~10 commands a Linux initiator actually

View File

@@ -33,7 +33,7 @@ The target drives login negotiation onto a single, fixed code path:
It implements SendTargets discovery, login (`AuthMethod=None` or one-way
**CHAP** — see [Authentication](#authentication-chap)), the ~10 SCSI
commands a Linux initiator issues to attach and use a block device
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`/`0xb0`, `REPORT LUNS`, `READ CAPACITY 10/16`,
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`, `REPORT LUNS`, `READ CAPACITY 10/16`,
`READ`/`WRITE 10/16`, `TEST UNIT READY`, `REQUEST SENSE`, `MODE SENSE`,
`SYNCHRONIZE CACHE`), the full R2T / Data-Out write path, NOP keepalive replies,
and Logout. Commands may be outstanding concurrently (Data-Out PDUs are routed
@@ -46,19 +46,6 @@ The package also ships a userspace **initiator** (`IscsiInitiator`) and an
Not implemented (yet): mutual (two-way) CHAP, Task Management (abort/reset),
target-initiated NOP keepalives, multiple connections/LUNs.
What a peer can make the target do is bounded:
- connections that have not logged in yet are limited in time (`loginTimeoutMs`)
and number (`maxPendingConnections`), and `allowedInitiatorAddresses` (the
source addresses of the expected initiators) closes the other ones as soon as
they are accepted, reporting them to `onConnectionRefused`
- a PDU data segment is at most 8 KiB during login, then the declared
`MaxRecvDataSegmentLength` (256 KiB), checked before it is read
- a single `READ`/`WRITE` is at most `maxTransferLength` (8 MiB by default),
advertised in the block limits VPD page so that initiators split their I/O
- a `BlockDevice` with `readOnly: true` is reported write-protected, and `WRITE`
is refused with `DATA PROTECT` before any data is solicited
### CLI
The package ships a `vates-iscsi` binary that serves a file as a LUN:
@@ -170,8 +157,6 @@ class RandomAccessDiskLun {
}
return out
}
// reported write-protected: WRITE is refused before reaching write()
readOnly = true
write() {
throw new Error('read-only LUN')
}

View File

@@ -35,11 +35,6 @@ VM boot mechanics, and deployment specifics live elsewhere and aren't reproduced
- Re-verified end-to-end after deploying the fix: same S3-backed boot scenario completed successfully
(~5 min total, vs ~20-25s for the equivalent local-storage boot — S3 itself is slow, but the mount no
longer wedges).
- Refined afterwards: the replacement used to happen as soon as the TCP connection was accepted, so any
peer reaching the port (a port scan, a failed CHAP login, a Discovery session) interrupted the session in
use. It now only happens once the new connection has established a Normal session; until then,
connections are bounded by `loginTimeoutMs` and `maxPendingConnections`. Regression tests:
`loopback.integ.mts`, `describe('connection admission', ...)`.
### Independently landed, cross-referenced against this pass's coverage

View File

@@ -1,6 +1,6 @@
{
"name": "@vates/iscsi",
"version": "0.4.0",
"version": "0.2.0",
"main": "dist/index.mjs",
"types": "dist/index.d.mts",
"bin": {
@@ -23,7 +23,7 @@
"dependencies": {
"@vates/async-each": "^1.0.3",
"@vates/read-chunk": "^1.2.1",
"@xen-orchestra/disk-transform": "^1.3.4",
"@xen-orchestra/disk-transform": "^1.3.0",
"@xen-orchestra/log": "^0.7.2",
"promise-toolbox": "^0.21.0"
},

View File

@@ -102,10 +102,10 @@ export class DiskBlockDevice implements BlockDevice {
return result
}
// the target refuses WRITE commands before they reach `write()`
readonly readOnly = true
/** Always throws: the source disk is read-only. */
/**
* Always throws: the source disk is read-only. The target answers CHECK
* CONDITION / MEDIUM ERROR, which initiators surface as an I/O error.
*/
async write(): Promise<void> {
throw new Error('DiskBlockDevice is read-only')
}

View File

@@ -1,129 +0,0 @@
import assert from 'node:assert/strict'
import { execFile, execFileSync } from 'node:child_process'
import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { promisify } from 'node:util'
import { after, before, describe, it } from 'node:test'
import { FileBlockDevice } from './index.mjs'
const execFileP = promisify(execFile)
const BLOCK_SIZE = 512
const SIZE = 4 * 1024 * 1024 // 4 MiB
/**
* Why this cannot run here, or `false` if it can. A real block device is the
* whole point, and the cheapest one to make is a loop device — which needs root
* and `losetup`.
*/
function skipReason(): string | false {
if (process.getuid === undefined || process.getuid() !== 0) {
return 'requires root to attach a loop device'
}
try {
execFileSync('losetup', ['--version'], { stdio: 'ignore' })
} catch {
return 'requires util-linux (losetup not found)'
}
return false
}
const bytePattern = (i: number) => (i * 31 + 11) & 0xff
describe('FileBlockDevice on a loop device', { skip: skipReason() }, () => {
let dir: string
let devicePath: string
before(async () => {
dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-loop-'))
const backing = join(dir, 'backing.img')
await writeFile(backing, '')
await truncate(backing, SIZE)
const { stdout } = await execFileP('losetup', ['--find', '--show', backing])
devicePath = stdout.trim()
})
after(async () => {
if (devicePath !== undefined) {
await execFileP('losetup', ['--detach', devicePath])
}
await rm(dir, { force: true, recursive: true })
})
const withDevice = async (fn: (device: FileBlockDevice) => Promise<void>): Promise<void> => {
const device = new FileBlockDevice({ path: devicePath, size: SIZE })
await device.open()
try {
await fn(device)
} finally {
await device.close()
}
}
it('accepts the device node and reports the capacity it was given', async () => {
await withDevice(async device => {
assert.equal(device.getSize(), SIZE)
assert.equal(device.getBlockSize(), BLOCK_SIZE)
})
})
it('round-trips block-aligned data', async () => {
await withDevice(async device => {
const payload = Buffer.alloc(8 * BLOCK_SIZE)
for (let i = 0; i < payload.length; i++) {
payload[i] = bytePattern(i)
}
await device.write(16 * BLOCK_SIZE, payload)
assert.deepEqual(await device.read(16 * BLOCK_SIZE, payload.length), payload)
})
})
it('round-trips data straddling block boundaries, at an unaligned offset', async () => {
await withDevice(async device => {
const payload = Buffer.from('a payload that is not a whole number of sectors long')
const offset = 3 * BLOCK_SIZE + 17
await device.write(offset, payload)
assert.deepEqual(await device.read(offset, payload.length), payload)
// the surrounding sector was read-modify-written by the kernel, not clobbered
assert.deepEqual(await device.read(3 * BLOCK_SIZE, 17), Buffer.alloc(17))
})
})
it('survives a reopen, which is what makes it a cache', async () => {
const payload = Buffer.alloc(BLOCK_SIZE, 0x5a)
await withDevice(device => device.write(2 * BLOCK_SIZE, payload))
await withDevice(async device => {
await device.flush()
assert.deepEqual(await device.read(2 * BLOCK_SIZE, BLOCK_SIZE), payload)
})
})
it('reads a large range in one call', async () => {
await withDevice(async device => {
const payload = Buffer.alloc(SIZE / 2)
for (let i = 0; i < payload.length; i++) {
payload[i] = bytePattern(i + 7)
}
await device.write(0, payload)
assert.deepEqual(await device.read(0, payload.length), payload)
})
})
it('throws on a read running past the end, instead of zero-filling it', async () => {
// the behaviour that sets the block device mode apart: a short read on a
// device of a known size is an I/O error, never a sparse tail
await withDevice(async device => {
await assert.rejects(device.read(SIZE, BLOCK_SIZE), /ended after 0 bytes/)
await assert.rejects(device.read(SIZE - BLOCK_SIZE, 2 * BLOCK_SIZE), /ended after 512 bytes/)
})
})
it('flushes without error', async () => {
await withDevice(async device => {
await device.write(0, Buffer.alloc(BLOCK_SIZE, 0xff))
await device.flush()
})
})
})

View File

@@ -16,11 +16,6 @@ export interface BlockDevice {
getSize(): number
/** Logical block size in bytes (typically 512). */
getBlockSize(): number
/**
* When true, the LUN is reported as write-protected and WRITE commands are
* refused up front (DATA PROTECT), without soliciting their data.
*/
readonly readOnly?: boolean
/** Read exactly `length` bytes starting at byte `offset`. */
read(offset: number, length: number): Promise<Buffer>
/** Write `data` starting at byte `offset`. */
@@ -34,60 +29,31 @@ export interface BlockDevice {
const DEFAULT_BLOCK_SIZE = 512
export interface FileBlockDeviceOptions {
/**
* Path to the backing file. It must already exist and be sized to capacity —
* or be a block device node, see {@link FileBlockDeviceOptions.size}.
*/
/** Path to the backing file. It must already exist and be sized to capacity. */
readonly path: string
/** Logical block size in bytes. Defaults to 512. */
readonly blockSize?: number
/**
* Capacity in bytes, for a block device node (e.g. a VDI hot-plugged onto the
* appliance's VM, `/dev/xvdX`), whose inode reports a size of 0.
*
* Setting it switches to the block device mode:
* - `path` must be a block device: a typo naming the wrong node is the
* difference between a cache and a destroyed disk;
* - a short read is an I/O error, not a sparse tail: zero-filling it would hand
* the initiator fabricated data.
*/
readonly size?: number
}
/**
* A {@link BlockDevice} backed by a regular (sparse) file, or a block device
* node, opened read/write.
* A {@link BlockDevice} backed by a regular (sparse) file opened read/write.
*
* The file must already exist and be sized to the desired capacity (e.g. via
* `fs.truncate`); this keeps the device a pure data store with no provisioning
* policy of its own.
*
* I/O is buffered, never `O_DIRECT`, which would need buffers aligned to the
* logical block size that Node cannot allocate.
*/
export class FileBlockDevice implements BlockDevice {
readonly #path: string
readonly #blockSize: number
readonly #isBlockDevice: boolean
#handle?: FileHandle
#size = 0
constructor({ path, blockSize = DEFAULT_BLOCK_SIZE, size }: FileBlockDeviceOptions) {
constructor({ path, blockSize = DEFAULT_BLOCK_SIZE }: FileBlockDeviceOptions) {
if (!Number.isInteger(blockSize) || blockSize <= 0) {
throw new Error(`blockSize must be a positive integer, got ${blockSize}`)
}
if (size !== undefined) {
if (!Number.isInteger(size) || size <= 0) {
throw new Error(`size must be a positive integer, got ${size}`)
}
if (size % blockSize !== 0) {
throw new Error(`size (${size}) is not a multiple of block size (${blockSize})`)
}
this.#size = size
}
this.#path = path
this.#blockSize = blockSize
this.#isBlockDevice = size !== undefined
}
/** Open the backing file and read its current size. Must be called before any I/O. */
@@ -97,18 +63,12 @@ export class FileBlockDevice implements BlockDevice {
}
const handle = await open(this.#path, 'r+')
try {
const stats = await handle.stat()
if (this.#isBlockDevice) {
if (!stats.isBlockDevice()) {
throw new Error(`${this.#path} is not a block device`)
}
} else {
if (stats.size % this.#blockSize !== 0) {
throw new Error(`backing file size (${stats.size}) is not a multiple of block size (${this.#blockSize})`)
}
this.#size = stats.size
const { size } = await handle.stat()
if (size % this.#blockSize !== 0) {
throw new Error(`backing file size (${size}) is not a multiple of block size (${this.#blockSize})`)
}
this.#handle = handle
this.#size = size
} catch (error) {
await handle.close()
throw error
@@ -139,10 +99,6 @@ export class FileBlockDevice implements BlockDevice {
while (read < length) {
const { bytesRead } = await handle.read(buffer, read, length - read, offset + read)
if (bytesRead === 0) {
if (this.#isBlockDevice) {
// the device is exactly as long as the caller said: the I/O failed
throw new Error(`read of ${length} bytes at ${offset} on ${this.#path} ended after ${read} bytes`)
}
// Reading past EOF (sparse tail): zero-fill the remainder.
buffer.fill(0, read)
break

View File

@@ -1,173 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { FileBlockDevice } from './backend.mjs'
const BLOCK_SIZE = 512
/** A temporary directory removed whatever the test does. */
const withTmpDir = async (fn: (dir: string) => Promise<void>): Promise<void> => {
const dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-backend-'))
try {
await fn(dir)
} finally {
await rm(dir, { force: true, recursive: true })
}
}
/** A sparse file of `size` bytes, as the class requires: existing and pre-sized. */
const makeFile = async (dir: string, size: number, name = 'lun.img'): Promise<string> => {
const path = join(dir, name)
await writeFile(path, '')
await truncate(path, size)
return path
}
describe('FileBlockDevice', () => {
it('rejects a non-positive-integer block size', () => {
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: 0 }), /positive integer/)
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: -1 }), /positive integer/)
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: 1.5 }), /positive integer/)
})
it('reads the capacity from the backing file', async () => {
await withTmpDir(async dir => {
const device = new FileBlockDevice({ path: await makeFile(dir, 4 * BLOCK_SIZE) })
await device.open()
try {
assert.equal(device.getSize(), 4 * BLOCK_SIZE)
assert.equal(device.getBlockSize(), BLOCK_SIZE)
} finally {
await device.close()
}
})
})
it('refuses a file whose size is not a multiple of the block size', async () => {
await withTmpDir(async dir => {
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE + 1) })
await assert.rejects(device.open(), /not a multiple of block size/)
})
})
it('throws on I/O before open()', async () => {
const device = new FileBlockDevice({ path: '/nope' })
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
await assert.rejects(device.write(0, Buffer.alloc(BLOCK_SIZE)), /must be called before I\/O/)
await assert.rejects(device.flush(), /must be called before I\/O/)
})
it('round-trips a write, at an offset and unaligned', async () => {
await withTmpDir(async dir => {
const device = new FileBlockDevice({ path: await makeFile(dir, 4 * BLOCK_SIZE) })
await device.open()
try {
const payload = Buffer.from('hello, lun')
await device.write(BLOCK_SIZE + 3, payload)
assert.deepEqual(await device.read(BLOCK_SIZE + 3, payload.length), payload)
// the bytes around it are untouched
assert.deepEqual(await device.read(BLOCK_SIZE, 3), Buffer.alloc(3))
} finally {
await device.close()
}
})
})
it('zero-fills the sparse tail of a read past EOF', async () => {
await withTmpDir(async dir => {
// a hole: nothing was ever written, so the whole file reads as zeroes
const device = new FileBlockDevice({ path: await makeFile(dir, 2 * BLOCK_SIZE) })
await device.open()
try {
assert.deepEqual(await device.read(0, 2 * BLOCK_SIZE), Buffer.alloc(2 * BLOCK_SIZE))
// past the end of the file, `fh.read` reports 0 bytes read: the remainder is
// zero-filled rather than being reported as an error
assert.deepEqual(await device.read(2 * BLOCK_SIZE, BLOCK_SIZE), Buffer.alloc(BLOCK_SIZE))
} finally {
await device.close()
}
})
})
it('reads back a large buffer in full', async () => {
await withTmpDir(async dir => {
const size = 1024 * BLOCK_SIZE
const device = new FileBlockDevice({ path: await makeFile(dir, size) })
await device.open()
try {
const payload = Buffer.alloc(size)
for (let i = 0; i < payload.length; i++) {
payload[i] = (i * 31 + 11) & 0xff
}
await device.write(0, payload)
assert.deepEqual(await device.read(0, size), payload)
} finally {
await device.close()
}
})
})
it('flushes and closes, and tolerates being opened or closed twice', async () => {
await withTmpDir(async dir => {
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE) })
await device.open()
await device.open() // no-op
await device.write(0, Buffer.alloc(BLOCK_SIZE, 0xaa))
await device.flush()
await device.close()
await device.close() // no-op
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
})
})
it('leaves no handle behind when open() fails', async () => {
await withTmpDir(async dir => {
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE + 1) })
await assert.rejects(device.open())
// the handle opened before the size check was closed again
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
})
})
})
// A real device node cannot be created without root, so what is covered here is
// everything around the I/O: the guards, the capacity, and the lifecycle. The
// I/O itself is covered by `backend.integ.mts`, against a loop device.
describe('FileBlockDevice with a size, over a block device', () => {
it('rejects a non-positive-integer size', () => {
for (const size of [0, -BLOCK_SIZE, 1.5]) {
assert.throws(() => new FileBlockDevice({ path: '/dev/null', size }), /size must be a positive integer/)
}
})
it('rejects a size that is not a multiple of the block size', () => {
assert.throws(
() => new FileBlockDevice({ path: '/dev/null', size: BLOCK_SIZE + 1 }),
/is not a multiple of block size/
)
})
it('reports the capacity it was given, never the inode', () => {
// nothing may be read from the inode, which for a real device node reports 0
const device = new FileBlockDevice({ path: '/dev/null', size: 4 * BLOCK_SIZE })
assert.equal(device.getSize(), 4 * BLOCK_SIZE)
assert.equal(device.getBlockSize(), BLOCK_SIZE)
})
it('refuses to open a path that is not a block device', async () => {
await withTmpDir(async dir => {
const path = await makeFile(dir, 4 * BLOCK_SIZE)
const device = new FileBlockDevice({ path, size: 4 * BLOCK_SIZE })
// the guard against a misconfigured path naming, say, a real disk's
// filesystem instead of the cache device
await assert.rejects(device.open(), new RegExp(`${path} is not a block device`))
// and the handle it opened to check was closed again
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
})
})
})

View File

@@ -24,13 +24,7 @@ import {
TEXT_FLAG_FINAL,
} from './constants.mjs'
import { buildFixedSense, handleScsiCommand, type ScsiIdentity } from './scsi.mjs'
import {
buildSendTargetsResponse,
LOGIN_MAX_DATA_SEGMENT_LENGTH,
LoginNegotiator,
OUR_MAX_RECV_DATA_SEGMENT_LENGTH,
parseTextKeys,
} from './login.mjs'
import { buildSendTargetsResponse, LoginNegotiator, parseTextKeys } from './login.mjs'
import { allocBhs, assemblePdu, type IncomingPdu, readPdu, writePdu } from './pdu.mjs'
import {
decodeCdb,
@@ -87,16 +81,6 @@ export interface ConnectionDeps {
allocateTsih(): number
/** When set, require one-way CHAP: the target challenges and verifies the initiator. */
readonly chap?: ChapCredentials
/**
* Time allowed to establish a Normal session, in ms (0 or unset disables):
* past it, the connection is closed. Also bounds Discovery sessions, which
* never become Normal ones.
*/
readonly loginTimeoutMs?: number
/** Called once this connection has established a Normal session. */
onSessionEstablished?(connection: Connection): void
/** Max length of a single READ or WRITE, in bytes. */
readonly maxTransferLength?: number
}
type Phase = 'login' | 'fullFeature' | 'closed'
@@ -139,8 +123,6 @@ export class Connection implements CommandContext {
readonly #pendingWrites = new Map<number, PendingWrite>()
#nextTargetTransferTag = 1
#loginTimer?: ReturnType<typeof setTimeout>
constructor(socket: Socket, deps: ConnectionDeps) {
this.#socket = socket
this.#deps = deps
@@ -160,19 +142,9 @@ export class Connection implements CommandContext {
/** Read and service PDUs until the peer logs out or the connection drops. */
async serve(): Promise<void> {
const { loginTimeoutMs = 0 } = this.#deps
if (loginTimeoutMs > 0) {
this.#loginTimer = setTimeout(() => {
log.warn('no session established in time, closing the connection', { remote: this.#socket.remoteAddress })
this.destroy()
}, loginTimeoutMs).unref()
}
try {
for (;;) {
const pdu = await readPdu(
this.#socket,
this.#phase === 'login' ? LOGIN_MAX_DATA_SEGMENT_LENGTH : OUR_MAX_RECV_DATA_SEGMENT_LENGTH
)
const pdu = await readPdu(this.#socket)
if (pdu === null) {
break // peer closed the connection
}
@@ -184,7 +156,6 @@ export class Connection implements CommandContext {
} catch (error) {
this.#fail(error)
} finally {
clearTimeout(this.#loginTimer)
this.#phase = 'closed'
this.#socket.destroy()
}
@@ -328,13 +299,6 @@ export class Connection implements CommandContext {
bhs[37] = statusDetail
await this.#send(assemblePdu(bhs, data))
// destroyed while the response was being written (login timeout, target
// closing): the write still succeeds, but this connection must not go on
// and replace the current session
if (this.#phase === 'closed') {
return true
}
if (statusClass !== LoginStatusClass.SUCCESS) {
// RFC 7143 §6.3: the failure response goes out first (above), then the
// target closes the connection.
@@ -352,10 +316,6 @@ export class Connection implements CommandContext {
sessionType: this.#sessionType,
target: this.#login.targetName,
})
if (this.#sessionType === 'Normal') {
clearTimeout(this.#loginTimer)
this.#deps.onSessionEstablished?.(this)
}
}
return false
}
@@ -429,13 +389,13 @@ export class Connection implements CommandContext {
// Not awaited: the read loop must move on immediately rather than block on
// this `lun.read()`. See the class doc for why no extra ordering is needed.
void this.#readGate
.run(() => handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity, this.#deps.maxTransferLength))
.run(() => handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity))
.catch(error => {
this.#fail(error)
})
return
}
await handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity, this.#deps.maxTransferLength)
await handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity)
}
async #sendReject(pdu: IncomingPdu, reason: number): Promise<void> {

View File

@@ -166,7 +166,6 @@ export const SenseKey = {
HARDWARE_ERROR: 0x04,
ILLEGAL_REQUEST: 0x05,
UNIT_ATTENTION: 0x06,
DATA_PROTECT: 0x07,
} as const
/** Additional Sense Code / Qualifier pairs we may emit. */
@@ -175,7 +174,6 @@ export const Asc = {
INVALID_COMMAND_OPERATION_CODE: { asc: 0x20, ascq: 0x00 },
INVALID_FIELD_IN_CDB: { asc: 0x24, ascq: 0x00 },
LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE: { asc: 0x21, ascq: 0x00 },
WRITE_PROTECTED: { asc: 0x27, ascq: 0x00 },
} as const
/** Reject reason codes (Reject PDU byte 2). */

View File

@@ -3,7 +3,7 @@ import { createLogger, type Logger } from '@xen-orchestra/log'
import type { BlockDevice } from './backend.mjs'
import { Connection, type ConnectionDeps } from './connection.mjs'
import { DEFAULT_MAX_TRANSFER_LENGTH, type ScsiIdentity } from './scsi.mjs'
import type { ScsiIdentity } from './scsi.mjs'
import type { ChapCredentials } from './types.mjs'
export { FileBlockDevice } from './backend.mjs'
@@ -25,8 +25,6 @@ const DEFAULT_PORT = 3260
const DEFAULT_WRITE_TIMEOUT_MS = 30_000
const DEFAULT_CMD_WINDOW = 64
const DEFAULT_READ_CONCURRENCY = 16
const DEFAULT_LOGIN_TIMEOUT_MS = 15_000
const DEFAULT_MAX_PENDING_CONNECTIONS = 8
const DEFAULT_IDENTITY: ScsiIdentity = {
vendor: 'VATES',
@@ -63,62 +61,18 @@ export interface IscsiTargetOptions {
* open-iscsi `node.session.auth` username/password). Omit for no authentication.
*/
readonly chap?: ChapCredentials
/**
* Time a new connection has to establish a Normal session, in ms (0
* disables). Also bounds Discovery sessions. Defaults to 15000.
*/
readonly loginTimeoutMs?: number
/**
* Max number of connections that have not established a Normal session yet;
* past it, new ones are refused. Defaults to 8.
*/
readonly maxPendingConnections?: number
/**
* Source addresses the initiators may connect from. When set, connections
* from any other address are closed as soon as they are accepted, before any
* login: only these initiators can take a pending connection slot.
* IPv4-mapped IPv6 addresses (`::ffff:a.b.c.d`) match their IPv4 form.
*/
readonly allowedInitiatorAddresses?: Iterable<string>
/**
* Called with the source address of each connection closed because it is
* not in `allowedInitiatorAddresses`, e.g. to tell why an initiator behind a
* NAT cannot connect.
*/
readonly onConnectionRefused?: (address: string) => void
/**
* Max length of a single READ or WRITE, in bytes, advertised in the block
* limits VPD page; longer commands are refused. At least one logical block.
* Defaults to 8 MiB.
*/
readonly maxTransferLength?: number
}
const IPV4_MAPPED_PREFIX = '::ffff:'
function normalizeAddress(address: string): string {
return address.startsWith(IPV4_MAPPED_PREFIX) && address.includes('.')
? address.slice(IPV4_MAPPED_PREFIX.length)
: address
}
/**
* A minimal iSCSI target exposing exactly one read/write LUN.
*
* Single initiator, single connection (`MaxConnections=1`),
* `ErrorRecoveryLevel=0`, no digests.
*
* A new connection replaces the current one rather than being refused (see
* `#onSessionEstablished`): there is no way to tell a stale, abandoned
* connection (initiator gave up without closing the socket) from a healthy
* one, so refusing would let one stuck connection wedge the target forever.
* The replacement only happens once the new connection has established a
* Normal session, CHAP included when configured, so that a port scan, a failed
* login or a Discovery session cannot interrupt the current one. Until then,
* connections are bounded in time (`loginTimeoutMs`) and in number
* (`maxPendingConnections`); with `allowedInitiatorAddresses`, only the expected
* initiators can take those slots. Not RFC 7143 session reinstatement (no ISID
* matching) — safe here since the target is ephemeral, CHAP-guarded, and
* `ErrorRecoveryLevel=0`, no digests. A new connection always replaces the
* current one rather than being refused (see `#onConnection`): there is no
* way to tell a stale, abandoned connection (initiator gave up without
* closing the socket) from a healthy one, so refusing would let one stuck
* connection wedge the target forever. Not RFC 7143 session reinstatement (no
* ISID matching) — safe here since the target is ephemeral, CHAP-guarded, and
* single-consumer.
*/
export class IscsiTarget {
@@ -131,17 +85,9 @@ export class IscsiTarget {
readonly #cmdWindow: number
readonly #readConcurrency: number
readonly #chap?: ChapCredentials
readonly #loginTimeoutMs: number
readonly #maxPendingConnections: number
readonly #allowedInitiatorAddresses?: ReadonlySet<string>
readonly #onConnectionRefused?: (address: string) => void
readonly #maxTransferLength: number
#server?: Server
// the connection with an established Normal session, serving the LUN
#connection?: Connection
// every open connection, so that close() can drop them all
readonly #connections = new Set<Connection>()
#tsih = 0
constructor(options: IscsiTargetOptions) {
@@ -153,16 +99,6 @@ export class IscsiTarget {
this.#cmdWindow = options.cmdWindow ?? DEFAULT_CMD_WINDOW
this.#readConcurrency = options.readConcurrency ?? DEFAULT_READ_CONCURRENCY
this.#chap = options.chap
this.#loginTimeoutMs = options.loginTimeoutMs ?? DEFAULT_LOGIN_TIMEOUT_MS
this.#maxPendingConnections = options.maxPendingConnections ?? DEFAULT_MAX_PENDING_CONNECTIONS
if (options.allowedInitiatorAddresses !== undefined) {
this.#allowedInitiatorAddresses = new Set(Array.from(options.allowedInitiatorAddresses, normalizeAddress))
}
this.#onConnectionRefused = options.onConnectionRefused
this.#maxTransferLength = options.maxTransferLength ?? DEFAULT_MAX_TRANSFER_LENGTH
if (this.#maxTransferLength < options.lun.getBlockSize()) {
throw new RangeError('maxTransferLength must be at least one logical block')
}
this.#identity = {
...DEFAULT_IDENTITY,
// Default the serial to the IQN so the LUN has a stable, unique identity.
@@ -186,44 +122,28 @@ export class IscsiTarget {
readConcurrency: this.#readConcurrency,
allocateTsih: () => this.#allocateTsih(),
chap: this.#chap,
loginTimeoutMs: this.#loginTimeoutMs,
onSessionEstablished: connection => this.#onSessionEstablished(connection),
maxTransferLength: this.#maxTransferLength,
}
}
#onConnection(socket: Socket): void {
const allowed = this.#allowedInitiatorAddresses
const remote = normalizeAddress(socket.remoteAddress ?? '')
if (allowed !== undefined && !allowed.has(remote)) {
log.warn('connection from an address that is not allowed, closing it', { remote })
socket.destroy()
this.#onConnectionRefused?.(remote)
return
const existing = this.#connection
if (existing !== undefined) {
// Replaces a possibly-stale connection rather than refusing; see the class doc.
log.warn('replacing existing connection with a new one', { remote: socket.remoteAddress })
existing.destroy()
}
socket.setNoDelay(true)
const connection = new Connection(socket, this.#deps())
this.#connections.add(connection)
this.#connection = connection
// Long-lived per-connection task; serve() handles its own errors and never
// rejects, so detaching it here is safe.
void connection.serve().finally(() => {
this.#connections.delete(connection)
if (this.#connection === connection) {
this.#connection = undefined
}
})
}
#onSessionEstablished(connection: Connection): void {
const existing = this.#connection
this.#connection = connection
if (existing !== undefined) {
// Replaces a possibly-stale connection rather than refusing; see the class doc.
log.warn('replacing existing connection with a newly logged in one')
existing.destroy()
}
}
/** Open the LUN (if needed) and start accepting connections. */
async listen(): Promise<void> {
if (this.#server !== undefined) {
@@ -232,11 +152,6 @@ export class IscsiTarget {
await this.#lun.open?.()
const server = createServer(socket => this.#onConnection(socket))
// + 1: the connection serving the LUN does not take a pending slot
server.maxConnections = this.#maxPendingConnections + 1
server.on('drop', data => {
log.warn('too many connections without a session, refusing a new one', { remote: data?.remoteAddress })
})
this.#server = server
try {
await new Promise<void>((resolve, reject) => {
@@ -260,20 +175,17 @@ export class IscsiTarget {
return address !== null && typeof address === 'object' ? address : undefined
}
/** Stop accepting connections, drop every connection, and close the LUN. */
/** Stop accepting connections, drop the active connection, and close the LUN. */
async close(): Promise<void> {
const server = this.#server
const connections = [...this.#connections]
const connection = this.#connection
this.#server = undefined
this.#connection = undefined
this.#connections.clear()
if (server !== undefined) {
await new Promise<void>((resolve, reject) => {
server.close(error => (error ? reject(error) : resolve()))
// Drop the live connections so server.close() can complete.
for (const connection of connections) {
connection.destroy()
}
// Drop the live connection so server.close() can complete.
connection?.destroy()
})
}
await this.#lun.close()

View File

@@ -35,12 +35,9 @@ const log: Logger = createLogger('vates:iscsi:login')
// Operational values the target imposes. The target drives negotiation, so these
// are pinned to a single code path (see plan): all writes are R2T-solicited
// (InitialR2T=Yes, ImmediateData=No), no digests, one connection.
export const OUR_MAX_RECV_DATA_SEGMENT_LENGTH = 262144
const OUR_MAX_RECV_DATA_SEGMENT_LENGTH = 262144
const OUR_MAX_BURST_LENGTH = 262144
const DEFAULT_INITIATOR_MAX_RECV_DATA_SEGMENT_LENGTH = 8192 // RFC 7143 default
// a declared MaxRecvDataSegmentLength only applies once logged in: until then,
// both sides keep to the 8192-byte default (RFC 7143 §13.12)
export const LOGIN_MAX_DATA_SEGMENT_LENGTH = 8192
/** Parse a NUL-delimited iSCSI text key=value data segment. */
export function parseTextKeys(data: Buffer): Map<string, string> {

View File

@@ -4,7 +4,6 @@ import { connect, type Socket } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { once } from 'node:events'
import { setTimeout as delay } from 'node:timers/promises'
import { after, before, describe, it } from 'node:test'
import {
@@ -643,200 +642,6 @@ describe('connection replacement', () => {
})
})
// A connection only takes over once it has itself logged in: until then, it
// must not disturb the session in use, whatever it does or fails to do.
describe('connection admission', () => {
const CHAP = { user: 'alice', secret: 's3cr3t-s3cr3t' }
let dir: string
let backingPath: string
before(async () => {
dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-admission-'))
backingPath = join(dir, 'lun.img')
await writeFile(backingPath, Buffer.alloc(LUN_SIZE, 0x5a))
})
after(async () => {
await rm(dir, { recursive: true, force: true })
})
async function withTarget(
options: {
loginTimeoutMs?: number
maxPendingConnections?: number
allowedInitiatorAddresses?: string[]
onConnectionRefused?: (address: string) => void
},
fn: (port: number) => Promise<void>
): Promise<void> {
const target = new IscsiTarget({
iqn: IQN,
host: '127.0.0.1',
port: 0,
lun: new FileBlockDevice({ path: backingPath, blockSize: BLOCK_SIZE }),
chap: CHAP,
...options,
})
await target.listen()
try {
const address = target.address()
assert.ok(address !== undefined)
await fn(address.port)
} finally {
await target.close()
}
}
async function rawConnect(port: number): Promise<Socket> {
const socket = connect(port, '127.0.0.1')
await once(socket, 'connect')
return socket
}
async function assertStillServing(initiator: IscsiInitiator): Promise<void> {
// leave the target time to act on whatever just happened
await delay(100)
assert.deepEqual(await initiator.read(0, BLOCK_SIZE), Buffer.alloc(BLOCK_SIZE, 0x5a))
}
it('a connection that sends nothing does not interrupt the session', async () => {
await withTarget({}, async port => {
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
const probe = await rawConnect(port)
try {
await assertStillServing(initiator)
} finally {
probe.destroy()
await initiator.close()
}
})
})
it('a failed CHAP login does not interrupt the session', async () => {
await withTarget({}, async port => {
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
try {
const intruder = new IscsiInitiator({
host: '127.0.0.1',
port,
targetIqn: IQN,
chap: { user: CHAP.user, secret: 'wrong-wrong-wrong' },
})
await assert.rejects(intruder.connect(), /login rejected/)
await assertStillServing(initiator)
} finally {
await initiator.close()
}
})
})
it('a Discovery session does not interrupt the session', async () => {
await withTarget({}, async port => {
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
const socket = await rawConnect(port)
try {
const discovery = new MiniInitiator(socket)
await discovery.login('Discovery')
assert.equal((await discovery.sendTargets()).get('TargetName'), IQN)
await assertStillServing(initiator)
await discovery.logout()
} finally {
socket.destroy()
await initiator.close()
}
})
})
it('an authenticated login replaces the session', async () => {
await withTarget({}, async port => {
const stale = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await stale.connect()
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
try {
await assert.rejects(stale.read(0, BLOCK_SIZE), /connection closed/)
await assertStillServing(initiator)
} finally {
await stale.close().catch(() => {})
await initiator.close()
}
})
})
it('closes a connection that does not log in in time', async () => {
await withTarget({ loginTimeoutMs: 100 }, async port => {
const probe = await rawConnect(port)
const closed = once(probe, 'close')
try {
await Promise.race([closed, delay(5000).then(() => assert.fail('the connection should have been closed'))])
} finally {
probe.destroy()
}
})
})
it('closes connections from an address that is not allowed, and reports it', async () => {
const refused: string[] = []
await withTarget(
{ allowedInitiatorAddresses: ['192.0.2.1'], onConnectionRefused: address => refused.push(address) },
async port => {
const probe = await rawConnect(port)
try {
await Promise.race([
once(probe, 'close'),
delay(5000).then(() => assert.fail('the connection should have been closed')),
])
} finally {
probe.destroy()
}
}
)
assert.deepEqual(refused, ['127.0.0.1'])
})
it('serves an allowed address, also in its IPv4-mapped IPv6 form', async () => {
for (const allowed of ['127.0.0.1', '::ffff:127.0.0.1']) {
await withTarget({ allowedInitiatorAddresses: [allowed] }, async port => {
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
try {
await assertStillServing(initiator)
} finally {
await initiator.close()
}
})
}
})
it('refuses connections without a session past the limit', async () => {
await withTarget({ maxPendingConnections: 2 }, async port => {
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
await initiator.connect()
const probes = [await rawConnect(port), await rawConnect(port)]
try {
const extra = await rawConnect(port)
probes.push(extra)
await Promise.race([
once(extra, 'close'),
delay(5000).then(() => assert.fail('the connection should have been refused')),
])
assert.equal(probes[0].destroyed, false)
assert.equal(probes[1].destroyed, false)
await assertStillServing(initiator)
} finally {
for (const probe of probes) {
probe.destroy()
}
await initiator.close()
}
})
})
})
// One loopback that exercises BOTH one-way CHAP roles at once: our target is the
// authenticator (challenges + verifies) and our initiator is the responder
// (answers the challenge). This is the same code path each production role uses

View File

@@ -86,12 +86,8 @@ export class IncomingPdu {
* No per-read timeout, intentionally — a target may sit idle between commands
* indefinitely. Liveness is a transport concern (socket timeout / NOP
* keepalives), handled elsewhere.
*
* A PDU announcing a data segment longer than `maxDataSegmentLength` is
* rejected before its payload is read, so a peer cannot make us buffer up to
* the 16 MiB the 24-bit length field allows.
*/
export async function readPdu(stream: Readable, maxDataSegmentLength = Infinity): Promise<IncomingPdu | null> {
export async function readPdu(stream: Readable): Promise<IncomingPdu | null> {
const bhs = await readChunk(stream, BHS_LENGTH)
if (bhs === null) {
return null // clean EOF between PDUs
@@ -102,9 +98,6 @@ export async function readPdu(stream: Readable, maxDataSegmentLength = Infinity)
const totalAhsLength = bhs.readUInt8(4)
const dataSegmentLength = bhs.readUIntBE(5, 3)
if (dataSegmentLength > maxDataSegmentLength) {
throw new Error(`iSCSI data segment too long (${dataSegmentLength} > ${maxDataSegmentLength} bytes)`)
}
const ahsBytes = totalAhsLength * 4
const dataBytesPadded = roundUp4(dataSegmentLength)

View File

@@ -56,24 +56,6 @@ describe('assemblePdu / readPdu round-trip', () => {
})
describe('readPdu framing', () => {
it('rejects a data segment longer than the limit before reading it', async () => {
const bhs = allocBhs(InitiatorOpcode.SCSI_COMMAND)
bhs.writeUIntBE(16 * 1024 * 1024 - 1, 5, 3) // announced data segment length
const stream = new PassThrough()
stream.write(bhs) // the announced payload is never sent
await assert.rejects(readPdu(stream, 8192), /data segment too long/)
})
it('accepts a data segment of exactly the limit', async () => {
const data = Buffer.alloc(8192, 0x5a)
const pdu = await (async () => {
const stream = new PassThrough()
stream.end(buildScsiCommandPdu(1, 1, Buffer.alloc(16), data))
return readPdu(stream, 8192)
})()
assert.deepEqual(pdu?.data, data)
})
it('reassembles a PDU split across multiple TCP chunks', async () => {
const data = Buffer.from('hello world!!') // 13 bytes -> padded to 16
const wire = buildScsiCommandPdu(7, 7, Buffer.alloc(16), data)

View File

@@ -40,20 +40,13 @@ export function buildStandardInquiry(identity: ScsiIdentity): Buffer {
return buffer
}
const SUPPORTED_VPD_PAGES = [0x00, 0x80, 0x83, 0xb0]
/** Default max length of a single READ or WRITE, in bytes. */
export const DEFAULT_MAX_TRANSFER_LENGTH = 8 * 1024 * 1024
const SUPPORTED_VPD_PAGES = [0x00, 0x80, 0x83]
/**
* Build an INQUIRY EVPD page, or `undefined` for an unsupported page code (the
* caller then returns CHECK CONDITION / INVALID FIELD IN CDB).
*/
export function buildVpd(
pageCode: number,
identity: ScsiIdentity,
maxTransferBlocks = DEFAULT_MAX_TRANSFER_LENGTH / 512
): Buffer | undefined {
export function buildVpd(pageCode: number, identity: ScsiIdentity): Buffer | undefined {
switch (pageCode) {
case 0x00: {
// Supported VPD pages.
@@ -90,15 +83,6 @@ export function buildVpd(
descriptor.copy(buffer, 4)
return buffer
}
case 0xb0: {
// Block limits: initiators split their I/O to the max transfer length
// instead of seeing it refused.
const buffer = Buffer.alloc(64)
buffer[1] = 0xb0
buffer.writeUInt16BE(0x3c, 2)
buffer.writeUInt32BE(maxTransferBlocks, 8) // maximum transfer length
return buffer
}
default:
return undefined
}
@@ -141,21 +125,15 @@ export function buildFixedSense({ key, asc, ascq }: SenseInfo): Buffer {
return buffer
}
// WP bit of the device-specific parameter in the mode parameter header
const MODE_SENSE_WRITE_PROTECT = 0x80
/** Minimal MODE SENSE parameter data: a header with no block descriptor or pages. */
export function buildModeSense(long: boolean, writeProtected = false): Buffer {
const deviceSpecific = writeProtected ? MODE_SENSE_WRITE_PROTECT : 0
export function buildModeSense(long: boolean): Buffer {
if (long) {
const buffer = Buffer.alloc(8)
buffer.writeUInt16BE(6, 0) // mode data length (8 - 2)
buffer[3] = deviceSpecific
return buffer
}
const buffer = Buffer.alloc(4)
buffer[0] = 3 // mode data length (4 - 1)
buffer[2] = deviceSpecific
return buffer
}
@@ -172,7 +150,6 @@ const ILLEGAL_REQUEST_LBA_OUT_OF_RANGE: SenseInfo = {
...Asc.LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE,
}
const MEDIUM_ERROR_NO_ADDITIONAL_SENSE: SenseInfo = { key: SenseKey.MEDIUM_ERROR, ...Asc.NO_ADDITIONAL_SENSE }
const DATA_PROTECT_WRITE_PROTECTED: SenseInfo = { key: SenseKey.DATA_PROTECT, ...Asc.WRITE_PROTECTED }
function checkCondition(ctx: CommandContext, itt: number, sense: SenseInfo): Promise<void> {
return ctx.sendScsiResponse(itt, ScsiStatus.CHECK_CONDITION, { sense: buildFixedSense(sense) })
@@ -184,22 +161,16 @@ function checkCondition(ctx: CommandContext, itt: number, sense: SenseInfo): Pro
* `cdb` is the 16-byte CDB from the SCSI Command PDU and `itt` its Initiator
* Task Tag. Read commands stream Data-In; WRITE solicits Data-Out via R2T;
* everything else returns status (with sense on error).
*
* A READ or WRITE longer than `maxTransferLength` bytes, advertised in the
* block limits VPD page, is refused with INVALID FIELD IN CDB: it also bounds
* the buffer a WRITE allocates.
*/
export async function handleScsiCommand(
cdb: Buffer,
itt: number,
ctx: CommandContext,
identity: ScsiIdentity,
maxTransferLength = DEFAULT_MAX_TRANSFER_LENGTH
identity: ScsiIdentity
): Promise<void> {
const { lun } = ctx
const blockSize = lun.getBlockSize()
const blockCount = lun.getSize() / blockSize
const maxTransferBlocks = Math.floor(maxTransferLength / blockSize)
const request = decodeCdb(cdb)
log.debug('scsi command', { itt, ...request })
@@ -212,7 +183,7 @@ export async function handleScsiCommand(
}
return ctx.sendReadData(itt, buildStandardInquiry(identity), request.allocationLength)
}
const page = buildVpd(request.pageCode, identity, maxTransferBlocks)
const page = buildVpd(request.pageCode, identity)
if (page === undefined) {
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
}
@@ -232,9 +203,6 @@ export async function handleScsiCommand(
if (request.lba + request.blocks > blockCount) {
return checkCondition(ctx, itt, ILLEGAL_REQUEST_LBA_OUT_OF_RANGE)
}
if (request.blocks > maxTransferBlocks) {
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
}
const byteLength = request.blocks * blockSize
if (byteLength === 0) {
return ctx.sendReadData(itt, Buffer.alloc(0), 0)
@@ -253,15 +221,9 @@ export async function handleScsiCommand(
}
case 'write': {
if (lun.readOnly === true) {
return checkCondition(ctx, itt, DATA_PROTECT_WRITE_PROTECTED)
}
if (request.lba + request.blocks > blockCount) {
return checkCondition(ctx, itt, ILLEGAL_REQUEST_LBA_OUT_OF_RANGE)
}
if (request.blocks > maxTransferBlocks) {
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
}
const byteLength = request.blocks * blockSize
if (byteLength === 0) {
return ctx.sendScsiResponse(itt, ScsiStatus.GOOD)
@@ -281,7 +243,7 @@ export async function handleScsiCommand(
}
case 'modeSense':
return ctx.sendReadData(itt, buildModeSense(request.long, lun.readOnly === true), request.allocationLength)
return ctx.sendReadData(itt, buildModeSense(request.long), request.allocationLength)
case 'syncCache':
await lun.flush()

View File

@@ -145,17 +145,9 @@ describe('response encoders', () => {
it('advertises supported VPD pages and rejects unknown ones', () => {
const page00 = buildVpd(0x00, IDENTITY)
assert.ok(page00 !== undefined)
assert.deepEqual([...page00.subarray(4)], [0x00, 0x80, 0x83, 0xb0])
assert.deepEqual([...page00.subarray(4)], [0x00, 0x80, 0x83])
assert.equal(buildVpd(0x55, IDENTITY), undefined)
})
it('advertises the max transfer length in the block limits page', () => {
const page = buildVpd(0xb0, IDENTITY, 2048)
assert.ok(page !== undefined)
assert.equal(page[1], 0xb0)
assert.equal(page.readUInt16BE(2), 0x3c)
assert.equal(page.readUInt32BE(8), 2048)
})
})
describe('handleScsiCommand', () => {
@@ -242,61 +234,4 @@ describe('handleScsiCommand', () => {
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
assert.equal(ctx.responses[0].options?.sense?.[12], 0x20) // INVALID COMMAND OPERATION CODE
})
for (const [name, opcode] of [
['READ(10)', 0x28],
['WRITE(10)', 0x2a],
] as const) {
it(`refuses a ${name} longer than the max transfer length, without reading or soliciting data`, async () => {
let reads = 0
const lun = new MemoryBlockDevice(8192)
const read = lun.read.bind(lun)
lun.read = (offset, length) => {
reads++
return read(offset, length)
}
const ctx = new FakeContext(lun)
const buffer = Buffer.alloc(16)
buffer[0] = opcode
buffer.writeUInt16BE(3, 7) // 3 blocks, the limit is 2
await handleScsiCommand(buffer, 1, ctx, IDENTITY, 1024)
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
assert.equal(ctx.responses[0].options?.sense?.[2], SenseKey.ILLEGAL_REQUEST)
assert.equal(ctx.responses[0].options?.sense?.[12], 0x24) // INVALID FIELD IN CDB
assert.equal(reads, 0)
assert.equal(ctx.writes.length, 0)
})
}
describe('read-only LUN', () => {
class ReadOnlyBlockDevice extends MemoryBlockDevice {
readonly readOnly = true
}
it('refuses WRITE(10) with DATA PROTECT, without soliciting its data', async () => {
const ctx = new FakeContext(new ReadOnlyBlockDevice(4096))
const buffer = Buffer.alloc(16)
buffer[0] = 0x2a
buffer.writeUInt16BE(1, 7)
await handleScsiCommand(buffer, 1, ctx, IDENTITY)
assert.equal(ctx.writes.length, 0)
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
assert.equal(ctx.responses[0].options?.sense?.[2], SenseKey.DATA_PROTECT)
assert.equal(ctx.responses[0].options?.sense?.[12], 0x27) // WRITE PROTECTED
})
it('reports write protection in MODE SENSE(6) and MODE SENSE(10)', async () => {
const ctx = new FakeContext(new ReadOnlyBlockDevice(4096))
await handleScsiCommand(cdb(0x1a, 0x00, 0x3f, 0x00, 255), 1, ctx, IDENTITY)
await handleScsiCommand(cdb(0x5a, 0x00, 0x3f, 0, 0, 0, 0, 0x01, 0x00), 2, ctx, IDENTITY)
assert.equal(ctx.reads[0].data[2] & 0x80, 0x80)
assert.equal(ctx.reads[1].data[3] & 0x80, 0x80)
})
it('does not report a writable LUN as write-protected', async () => {
const ctx = new FakeContext(new MemoryBlockDevice(4096))
await handleScsiCommand(cdb(0x1a, 0x00, 0x3f, 0x00, 255), 1, ctx, IDENTITY)
assert.equal(ctx.reads[0].data[2] & 0x80, 0)
})
})
})

View File

@@ -56,21 +56,17 @@ function parseAttachedDisk(sessionOutput: string): string | undefined {
}
/**
* Neither the device node nor a nonzero `/sys/class/block/<name>/size` means
* the disk can be opened: the sd driver sets the size before
* `device_add_disk()`, which creates the sysfs entry, then the `/dev` node, then
* registers the block device: until then `open()` fails with ENOENT or ENXIO.
* Only an `open()` that succeeds means ready.
* `stat()`ing the device node only proves udev created it, not that the SCSI
* layer has finished attaching it — `open()` right after can still fail with
* ENXIO. `/sys/class/block/<name>/size` reporting nonzero is what actually
* means ready, the same check `_cache.mjs`'s `openLocalDevice` uses for the
* same reason on XAPI-plugged devices.
*/
async function isDeviceReady(name: string): Promise<boolean> {
try {
await (await openFile(`/dev/${name}`, 'r')).close()
return true
} catch (error) {
if (error instanceof Error && 'code' in error && (error.code === 'ENXIO' || error.code === 'ENOENT')) {
return false
}
throw error
return Number.parseInt(await readFile(`/sys/class/block/${name}/size`, 'utf8'), 10) > 0
} catch {
return false
}
}

View File

@@ -54,11 +54,6 @@ export default class AbstractNbdClient {
// publish itself
#connectGeneration = 0
// generation of the attempt which published the current transport, undefined
// when not connected: an error on a transport of another generation is stale
// and must be ignored
#connectedGeneration
#waitBeforeReconnect
#readBlockRetries
#reconnectRetry
@@ -72,6 +67,7 @@ export default class AbstractNbdClient {
#nextCommandQueryId = BigInt(0)
// map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
#commandQueryBacklog = new Map()
#connected = false
#reconnectingPromise
@@ -112,7 +108,7 @@ export default class AbstractNbdClient {
}
get connected() {
return this.#connectedGeneration !== undefined
return this.#connected
}
/* ---------------------------------------------------------------------------
@@ -206,31 +202,22 @@ export default class AbstractNbdClient {
* connection
* ------------------------------------------------------------------------- */
#onTransportError(generation, error) {
#onTransportError = error => {
// without this listener an error on the transport would be thrown as an
// uncaught exception, and pending reads would only fail on message timeout
//
// an error on a transport which is not the current one is expected (a
// closed transport can be reset late by the other end, after a reconnection)
// and must not reject the reads of the current transport
//
// an error during the handshake is reported by the handshake reads
// themselves, through the connect() rejection
if (generation !== this.#connectedGeneration) {
debug('error on a stale nbd transport', { error })
return
}
warn('error on the nbd transport', { error })
// an error outside of the connected window is expected (the other end can
// be gone already while we're closing), and always reported to the caller
// through the connect()/readBlock() rejection
const log = this.#connected ? warn : debug
log('error on the nbd transport', { error })
this.#rejectAll(error)
}
// the generation is used instead of the transport itself since an attempt can
// own multiple transports (the raw one and the one upgraded to TLS)
#watchTransport(transport, generation) {
const onError = error => this.#onTransportError(generation, error)
transport.readable.on('error', onError)
#watchTransport(transport) {
transport.readable.on('error', this.#onTransportError)
if (transport.writable !== transport.readable) {
transport.writable.on('error', onError)
transport.writable.on('error', this.#onTransportError)
}
}
@@ -240,10 +227,10 @@ export default class AbstractNbdClient {
// not cancel on timeout, so a superseded attempt must not be able to use,
// replace or destroy the transport of the attempt which replaced it
let transport = await this._openTransport()
this.#watchTransport(transport, generation)
this.#watchTransport(transport)
try {
// the transport can be replaced during the handshake (TLS upgrade)
transport = await this.#handshake(transport, generation)
transport = await this.#handshake(transport)
if (generation !== this.#connectGeneration) {
const error = new Error('this connection has been superseded by a newer one')
@@ -257,7 +244,7 @@ export default class AbstractNbdClient {
throw error
}
this.#transport = transport
this.#connectedGeneration = generation
this.#connected = true
// reset internal state if we reconnected a nbd client
this.#commandQueryBacklog = new Map()
this.#waitingForResponse = false
@@ -280,10 +267,10 @@ export default class AbstractNbdClient {
}
async disconnect() {
if (this.#connectedGeneration === undefined) {
if (!this.#connected) {
return
}
this.#connectedGeneration = undefined
this.#connected = false
const transport = this.#transport
this.#transport = undefined
if (transport === undefined) {
@@ -379,10 +366,9 @@ export default class AbstractNbdClient {
//
/**
* @param {NbdTransport} transport
* @param {number} generation - the connection attempt owning the transport
* @returns {Promise<NbdTransport>}
*/
async #handshake(transport, generation) {
async #handshake(transport) {
assert((await this.#read(transport, 8)).equals(INIT_PASSWD))
assert((await this.#read(transport, 8)).equals(OPTS_MAGIC))
const flagsBuffer = await this.#read(transport, 2)
@@ -393,7 +379,7 @@ export default class AbstractNbdClient {
// let the subclass upgrade the transport if it needs to (TLS)
const secured = await this._secureTransport(transport)
if (secured !== transport) {
this.#watchTransport(secured, generation)
this.#watchTransport(secured)
transport = secured
}

View File

@@ -166,28 +166,6 @@ describe('AbstractNbdClient', () => {
}
})
it('does not let an error on a superseded transport disturb the current one', async () => {
const client = new InMemoryNbdClient({}, { readBlockRetries: 1, waitBeforeReconnect: 0 })
await client.connect()
await client.reconnect()
try {
assert.equal(client.transports.length, 2)
const [supersededTransport] = client.transports
// the read is queued synchronously, the in memory server answers later
const promise = client.readBlock(0, BLOCK_SIZE)
// the other end of the closed transport resets it late
supersededTransport.readable.emit('error', Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))
const block = await promise
assert.ok(block.equals(DATA.subarray(0, BLOCK_SIZE)))
// no reconnection has been triggered
assert.equal(client.transports.length, 2)
} finally {
await client.disconnect()
}
})
it('does not support getMap()', async () => {
const client = new InMemoryNbdClient()
await assert.rejects(client.getMap(), ({ code }) => code === 'NBD_MAP_UNSUPPORTED')

View File

@@ -1,7 +1,4 @@
import { once } from 'node:events'
import { request as httpRequest } from 'node:http'
import { request as httpsRequest } from 'node:https'
import { isIPv6, Socket } from 'node:net'
import { Socket } from 'node:net'
import { connect } from 'node:tls'
import { spawn } from 'node:child_process'
import AbstractNbdClient from './AbstractNbdClient.mjs'
@@ -14,9 +11,6 @@ import { NBD_DEFAULT_PORT, NBD_OPT_STARTTLS } from './constants.mjs'
* @extends {AbstractNbdClient}
*/
export default class NbdTcpClient extends AbstractNbdClient {
#httpProxy
#proxyRejectUnauthorized
#proxyTimeout
#serverAddress
#serverCert
#serverPort
@@ -27,103 +21,18 @@ export default class NbdTcpClient extends AbstractNbdClient {
* @param {number} [settings.port]
* @param {string} [settings.exportname]
* @param {string} [settings.cert] - PEM certificate, enables TLS when set
* @param {string} [settings.httpProxy] - URL of an HTTP(S) proxy, the connection is tunneled through it with `CONNECT`
* @param {boolean} [settings.proxyRejectUnauthorized=true] - whether to check the certificate of an HTTPS proxy
* @param {number} [settings.proxyTimeout=6e4] - delay in ms after which an unanswered `CONNECT` is aborted
* @param {object} [options] - see {@link AbstractNbdClient}
*/
constructor(
{
address,
port = NBD_DEFAULT_PORT,
exportname,
cert,
httpProxy,
proxyRejectUnauthorized = true,
proxyTimeout = 6e4,
},
options
) {
constructor({ address, port = NBD_DEFAULT_PORT, exportname, cert }, options) {
super({ exportname }, options)
// other proxy protocols (e.g. SOCKS) are not supported for now
if (httpProxy !== undefined) {
const { protocol } = new URL(httpProxy)
if (protocol !== 'http:' && protocol !== 'https:') {
const error = new Error(`unsupported proxy protocol ${protocol}, only http: and https: are supported`)
error.code = 'NBD_PROXY_UNSUPPORTED_PROTOCOL'
throw error
}
}
this.#serverAddress = address
this.#serverPort = port
this.#serverCert = cert
this.#httpProxy = httpProxy
this.#proxyRejectUnauthorized = proxyRejectUnauthorized
this.#proxyTimeout = proxyTimeout
}
/**
* Open a raw TCP tunnel to the NBD server through an HTTP proxy
*
* https://datatracker.ietf.org/doc/html/rfc9110#section-9.3.6
*
* @returns {Promise<import('node:net').Socket>}
*/
async #connectThroughHttpProxy() {
const proxy = new URL(this.#httpProxy)
const isHttps = proxy.protocol === 'https:'
const address = this.#serverAddress
const target = `${isIPv6(address) ? `[${address}]` : address}:${this.#serverPort}`
const headers = { host: target }
if (proxy.username !== '' || proxy.password !== '') {
const credentials = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`
headers['proxy-authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`
}
const req = (isHttps ? httpsRequest : httpRequest)({
agent: false,
headers,
// URL keeps the brackets around IPv6 addresses
hostname: proxy.hostname.replace(/^\[(.*)\]$/, '$1'),
method: 'CONNECT',
path: target,
port: proxy.port,
rejectUnauthorized: this.#proxyRejectUnauthorized,
timeout: this.#proxyTimeout,
})
// `timeout` only emits an event, the request must be aborted explicitly
req.on('timeout', () => {
const error = new Error(`HTTP proxy did not answer the CONNECT to ${target} in time`)
error.code = 'NBD_PROXY_CONNECT_TIMEOUT'
req.destroy(error)
})
req.end()
// the NBD server may already have sent some data, it is in `head`
const [res, socket, head] = await once(req, 'connect')
// the tunnel is established, NBD has its own timeouts
socket.setTimeout(0)
if (res.statusCode !== 200) {
socket.destroy()
const error = new Error(`HTTP proxy refused to connect to ${target}: ${res.statusCode} ${res.statusMessage}`)
error.code = 'NBD_PROXY_CONNECT_FAILED'
throw error
}
if (head.length !== 0) {
socket.unshift(head)
}
return socket
}
// mandatory , at least to start the handshake: the connection always starts
// unsecured, and is upgraded to TLS during the handshake
async _openTransport() {
if (this.#httpProxy !== undefined) {
const socket = await this.#connectThroughHttpProxy()
return { readable: socket, writable: socket }
}
const socket = new Socket()
await new Promise((resolve, reject) => {
socket.connect(this.#serverPort, this.#serverAddress)

View File

@@ -1,107 +0,0 @@
import assert from 'node:assert/strict'
import { after, before, describe, it } from 'node:test'
import { createServer as createHttpServer } from 'node:http'
import { connect, createServer as createTcpServer } from 'node:net'
import { once } from 'node:events'
import NbdTcpClient from './NbdTcpClient.mjs'
import { serveNbd } from './tests/fake-nbd-server.mjs'
const BLOCK_SIZE = 64 * 1024
const DATA = Buffer.alloc(4 * BLOCK_SIZE)
for (let i = 0; i < DATA.length; i++) {
DATA[i] = i % 251
}
const CREDENTIALS = 'Basic ' + Buffer.from('user:p@ss').toString('base64')
describe('NbdTcpClient through an HTTP proxy', () => {
let nbdServer, proxyServer, nbdPort, proxyPort
const connectRequests = []
before(async () => {
nbdServer = createTcpServer(socket => {
serveNbd({ readable: socket, writable: socket, data: DATA }).catch(() => {})
})
nbdServer.listen(0, '127.0.0.1')
await once(nbdServer, 'listening')
nbdPort = nbdServer.address().port
// minimal CONNECT proxy, similar to the XO Proxy one
proxyServer = createHttpServer()
proxyServer.on('connect', (req, clientSocket, head) => {
connectRequests.push(req.url)
if (req.headers['proxy-authorization'] !== CREDENTIALS) {
clientSocket.end('HTTP/1.1 407 Proxy Authentication Required\r\n\r\n')
return
}
const { hostname, port } = new URL('http://' + req.url)
const serverSocket = connect(port, hostname, () => {
clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
serverSocket.write(head)
serverSocket.pipe(clientSocket).pipe(serverSocket)
})
serverSocket.on('error', () => clientSocket.destroy())
clientSocket.on('error', () => serverSocket.destroy())
})
proxyServer.listen(0, '127.0.0.1')
await once(proxyServer, 'listening')
proxyPort = proxyServer.address().port
})
after(() => {
proxyServer.closeAllConnections()
proxyServer.close()
nbdServer.close()
})
it('tunnels the NBD connection with CONNECT', async () => {
const client = new NbdTcpClient({
address: '127.0.0.1',
port: nbdPort,
httpProxy: `http://user:p%40ss@127.0.0.1:${proxyPort}`,
})
await client.connect()
try {
assert.equal(client.exportSize, BigInt(DATA.length))
const block = await client.readBlock(2, BLOCK_SIZE)
assert.deepEqual(block, DATA.subarray(2 * BLOCK_SIZE, 3 * BLOCK_SIZE))
} finally {
await client.disconnect()
}
assert.equal(connectRequests.at(-1), `127.0.0.1:${nbdPort}`)
})
it('fails when the proxy refuses the connection', async () => {
const client = new NbdTcpClient({
address: '127.0.0.1',
port: nbdPort,
httpProxy: `http://user:wrong@127.0.0.1:${proxyPort}`,
})
await assert.rejects(client.connect(), { code: 'NBD_PROXY_CONNECT_FAILED' })
await client.disconnect().catch(() => {})
})
it('aborts the CONNECT when the proxy does not answer', async () => {
// accepts the TCP connection but never answers
const silentProxy = createTcpServer(() => {})
silentProxy.listen(0, '127.0.0.1')
await once(silentProxy, 'listening')
try {
const client = new NbdTcpClient({
address: '127.0.0.1',
port: nbdPort,
httpProxy: `http://127.0.0.1:${silentProxy.address().port}`,
proxyTimeout: 50,
})
await assert.rejects(client.connect(), { code: 'NBD_PROXY_CONNECT_TIMEOUT' })
} finally {
silentProxy.close()
}
})
it('rejects unsupported proxy protocols', () => {
assert.throws(() => new NbdTcpClient({ address: '127.0.0.1', httpProxy: 'socks5://127.0.0.1:1080' }), {
code: 'NBD_PROXY_UNSUPPORTED_PROTOCOL',
})
})
})

View File

@@ -13,7 +13,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "3.7.0",
"version": "3.6.0",
"engines": {
"node": ">=14.0"
},
@@ -22,10 +22,10 @@
"@vates/async-each": "^1.0.3",
"@vates/read-chunk": "^1.2.1",
"@xen-orchestra/async-map": "^0.1.5",
"@xen-orchestra/disk-transform": "^1.3.4",
"@xen-orchestra/disk-transform": "^1.3.3",
"@xen-orchestra/log": "^0.7.2",
"promise-toolbox": "^0.21.0",
"xen-api": "^5.1.0"
"xen-api": "^5.0.1"
},
"devDependencies": {
"tap": "^18.7.0",

View File

@@ -1,7 +1,7 @@
{
"name": "@vates/types",
"private": false,
"version": "2.5.0",
"version": "2.3.0",
"main": "./dist/index.mjs",
"exports": {
".": {
@@ -11,10 +11,6 @@
"./xo": "./dist/xo.d.mts",
"./xo-app": "./dist/xo-app.d.mts",
"./xen-api": "./dist/xen-api.d.mts",
"./backup-expression-context": {
"default": "./dist/backup-expression-context.mjs",
"types": "./dist/backup-expression-context.d.mts"
},
"./common": {
"default": "./dist/common.mjs",
"types": "./dist/common.d.mts"
@@ -38,7 +34,7 @@
"typescript": "~5.6.3",
"typescript-eslint": "^8.61.0",
"@types/express": "^5.0.0",
"@xen-orchestra/disk-transform": "^1.3.4"
"@xen-orchestra/disk-transform": "^1.3.3"
},
"scripts": {
"build": "tsc",

View File

@@ -1,38 +0,0 @@
export const BACKUP_EXPRESSION_CONTEXT = {
run: {
dayOfWeek: { type: 'number' },
dayOfMonth: { type: 'number' },
hour: { type: 'number' },
month: { type: 'number' },
year: { type: 'number' },
},
sr: {
name_label: { type: 'string' },
type: { type: 'string' },
description: { type: 'string' },
tags: { type: 'string[]' },
},
backupRepository: {
name: { type: 'string' },
type: { type: 'string' },
tags: { type: 'string[]' },
},
vm: {
name_label: { type: 'string' },
tags: { type: 'string[]' },
power_state: { type: 'string' },
},
chainLength: { type: 'number' },
} as const
export type BACKUP_EXPRESSION_CONTEXT = (typeof BACKUP_EXPRESSION_CONTEXT)[keyof typeof BACKUP_EXPRESSION_CONTEXT]
export const BACKUP_SETTING_CONTEXT = {
healthCheckVmsWithTags: ['vm', 'run'],
fullInterval: ['vm', 'run', 'chainLength'],
vmFilter: ['vm', 'run'],
srs: ['sr', 'run'],
remotes: ['backupRepository', 'run'],
} as const
export type BACKUP_SETTING_CONTEXT = (typeof BACKUP_SETTING_CONTEXT)[keyof typeof BACKUP_SETTING_CONTEXT]

View File

@@ -663,19 +663,6 @@ export const SUPPORTED_VDI_FORMAT = {
export type SUPPORTED_VDI_FORMAT = (typeof SUPPORTED_VDI_FORMAT)[keyof typeof SUPPORTED_VDI_FORMAT]
/**
* Formats a VDI can be exported to by XO: the XAPI ones, plus the ones XO generates
* itself from the blocks of the disk.
*/
export const SUPPORTED_VDI_EXPORT_FORMAT = {
raw: 'raw',
vhd: 'vhd',
qcow2: 'qcow2',
vmdk: 'vmdk',
} as const
export type SUPPORTED_VDI_EXPORT_FORMAT = (typeof SUPPORTED_VDI_EXPORT_FORMAT)[keyof typeof SUPPORTED_VDI_EXPORT_FORMAT]
// ----- XAPI Stats
type XapiStatsResponse<T> = {
@@ -772,7 +759,6 @@ export type RawTrafficRule = {
ipRange: string
direction: TrafficRuleDirection
port?: string
priority?: number
}
type BaseTrafficRule = RawTrafficRule & {
@@ -814,7 +800,3 @@ export const XAPI_TYPES: readonly string[] = [
export const RPU_RECOVERY_STEP_NAMES = ['evacuate', 'update', 'reboot', 'enable', 'restoreVms'] as const
export type RPU_RECOVERY_STEP_NAME = (typeof RPU_RECOVERY_STEP_NAMES)[number]
export const COMMON_TAG = {
NOBAK: 'xo:no-bak',
} as const

View File

@@ -4,8 +4,5 @@ export * from './xo.mjs'
export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/xen-orchestra-acl.mjs'
export * from './lib/xen-orchestra-plugins.mjs'
export * from './lib/xen-orchestra-v2v.mjs'
export * from './lib/complex-matcher.mjs'
export * from './lib/rest-api.mjs'
export * from './lib/xen-orchestra-vmware-explorer.mjs'

File diff suppressed because it is too large Load Diff

View File

@@ -1,219 +0,0 @@
import type { Branded } from '../common.mjs'
import type { XoNetwork, XoSr, XoVmTemplate } from '../xo.mjs'
// Stateless API: requests needing the source carry the connection in a POST body (never in URLs or access logs) and
// xo-server stores nothing it reads from the source.
export const V2V_PLATFORM = {
VMWARE: 'vmware',
} as const
export type V2V_PLATFORM = (typeof V2V_PLATFORM)[keyof typeof V2V_PLATFORM]
// 422 rather than 401 for invalid source credentials: a 401 would open the browser Basic authentication prompt
export const V2V_ERROR_CODE = {
INVALID_CREDENTIALS: 'V2V_INVALID_CREDENTIALS',
} as const
export type V2V_ERROR_CODE = (typeof V2V_ERROR_CODE)[keyof typeof V2V_ERROR_CODE]
// Clones, restores and replicas inherit `other_config`: the imported VM records its own UUID to be told apart from
// them, and the source identity because a moref is only unique within one vCenter. The source identity is the
// inventory `instanceUuid`, or `<lowercased host>:<port>` for a standalone ESXi which has none.
export const V2V_VM_OTHER_CONFIG = {
SOURCE_VM_ID: 'sourceVmId',
SOURCE_SNAPSHOT_ID: 'sourceSnapshotId',
IMPORTED_VM_UUID: 'xo:v2v:importedVmUuid',
SOURCE_ID: 'xo:v2v:sourceId',
SOURCE_VM_INSTANCE_UUID: 'xo:v2v:sourceVmInstanceUuid',
} as const
export type V2V_VM_OTHER_CONFIG = (typeof V2V_VM_OTHER_CONFIG)[keyof typeof V2V_VM_OTHER_CONFIG]
export type XoV2vVmOtherConfig = Partial<Record<V2V_VM_OTHER_CONFIG, string>>
// The disk of the source chain an imported VDI last received, one of the `chainPaths` of its source disk: the CID
// cannot tell a fresh snapshot delta from its parent, the path can
export const V2V_VDI_OTHER_CONFIG = {
DISK_PATH: 'esxi_diskPath',
} as const
export type V2V_VDI_OTHER_CONFIG = (typeof V2V_VDI_OTHER_CONFIG)[keyof typeof V2V_VDI_OTHER_CONFIG]
export const V2V_START_BLOCKED_REASON = 'Esxi migration in progress...'
export const V2V_PARTIAL_IMPORT_TAG = 'xo:no-bak=V2V partial import'
export const V2V_SNAPSHOT_NAME = {
AFTER_PARTIAL_IMPORT: 'after partial import from V2V',
AFTER_COMPLETE_IMPORT: 'after complete import from V2V',
COMPLETE_IMPORT: 'complete import from V2V',
} as const
export type V2V_SNAPSHOT_NAME = (typeof V2V_SNAPSHOT_NAME)[keyof typeof V2V_SNAPSHOT_NAME]
// Set on the task of a start so the front can tell which source VMs are being imported
export const V2V_TASK_PROPERTY = {
SOURCE_VM_IDS: 'sourceVmIds',
} as const
export type XoV2vConnection = {
host: string
port: number
username: string
password: string
skipSslCheck: boolean
}
export type XoV2vStatus = {
vectura: { status: 'success' } | { status: 'error'; error: string }
}
export type XoV2vInventoryPayload = {
platform: V2V_PLATFORM
connection: XoV2vConnection
}
export type XoV2vDatacenter = {
id: Branded<'v2v-datacenter'>
name: string
}
export type XoV2vCluster = {
id: Branded<'v2v-cluster'>
name: string
datacenterId: XoV2vDatacenter['id']
}
export type XoV2vHost = {
id: Branded<'v2v-host'>
name: string
clusterId?: XoV2vCluster['id']
datacenterId: XoV2vDatacenter['id']
}
// Sizes are in bytes, as in the rest of XO, not in the KB/MB VMware returns
export type XoV2vDatastore = {
id: Branded<'v2v-datastore'>
name: string
type: string
capacity: number
free: number
hostIds: XoV2vHost['id'][]
}
export type XoV2vPortGroup = {
id: Branded<'v2v-port-group'>
name: string
vlan?: number
}
export const V2V_DISK_MODE = {
PERSISTENT: 'persistent',
NON_PERSISTENT: 'nonpersistent',
INDEPENDENT_PERSISTENT: 'independent_persistent',
INDEPENDENT_NON_PERSISTENT: 'independent_nonpersistent',
UNDOABLE: 'undoable',
APPEND: 'append',
} as const
export type V2V_DISK_MODE = (typeof V2V_DISK_MODE)[keyof typeof V2V_DISK_MODE]
// `key` is the vmx device node (`scsi0:0`, `ethernet0`…), the identifier the import matches the start maps against
export type XoV2vDisk = {
key: Branded<'v2v-disk'>
label: string
datastoreId: XoV2vDatastore['id']
capacity: number
mode: V2V_DISK_MODE
isRdm: boolean
isEncrypted: boolean
// `dir/disk.vmdk` in its datastore (no `[datastore]` prefix) of the base disk, then of each snapshot delta down to
// the active disk: the chain the import walks for this vmx node
chainPaths: string[]
}
// No port group for opaque (NSX) or missing backings
export type XoV2vNic = {
key: Branded<'v2v-nic'>
label: string
portGroupId?: XoV2vPortGroup['id']
macAddress: string
}
export const V2V_VM_POWER_STATE = {
POWERED_ON: 'poweredOn',
POWERED_OFF: 'poweredOff',
SUSPENDED: 'suspended',
} as const
export type V2V_VM_POWER_STATE = (typeof V2V_VM_POWER_STATE)[keyof typeof V2V_VM_POWER_STATE]
export const V2V_FIRMWARE = {
BIOS: 'bios',
UEFI: 'uefi',
} as const
export type V2V_FIRMWARE = (typeof V2V_FIRMWARE)[keyof typeof V2V_FIRMWARE]
export type XoV2vVm = {
id: Branded<'v2v-vm'>
instanceUuid: string
name: string
description: string
powerState: V2V_VM_POWER_STATE
hostId: XoV2vHost['id']
nCpus: number
memory: number
firmware: V2V_FIRMWARE
secureBoot: boolean
datastoreIds: XoV2vDatastore['id'][]
snapshotCount: number
// Compared with `other_config.sourceSnapshotId` of a partial import: a delta is only possible while they match
currentSnapshotId?: string
disks: XoV2vDisk[]
nics: XoV2vNic[]
}
export type XoV2vInventory = {
product: string
version: string
// Absent on a standalone ESXi, identified by its connection address instead
instanceUuid?: string
datacenters: XoV2vDatacenter[]
clusters: XoV2vCluster[]
hosts: XoV2vHost[]
datastores: XoV2vDatastore[]
portGroups: XoV2vPortGroup[]
vms: XoV2vVm[]
}
export type XoV2vStartOptions = {
stopSource: boolean
startVm: boolean
cleanup: boolean
}
// A resumed import sends no disk or NIC map: the back end reads its mapping from the existing XCP-ng VM
export type XoV2vStartPayloadVm = Partial<XoV2vStartOptions> & {
id: XoV2vVm['id']
disks?: Record<XoV2vDisk['key'], XoSr['id']>
nics?: Record<XoV2vNic['key'], XoNetwork['id']>
template?: XoVmTemplate['id']
tags: string[]
keepMac: boolean
}
// Body of `POST /pools/{id}/actions/import_vms_from_vmware`: the pool is in the path. Checked against the body of the
// route when the REST API is built, which refuses any extra key
export type XoV2vStartPayload = {
connection: XoV2vConnection
defaults: XoV2vStartOptions & {
sr: XoSr['id']
network: XoNetwork['id']
template: XoVmTemplate['id']
}
vms: XoV2vStartPayloadVm[]
stopOnError: boolean
concurrency: number
}

View File

@@ -1,103 +0,0 @@
// shape of `Esxi#getInventory()` in `@xen-orchestra/vmware-explorer`
// the objects reference each other through their VMware managed object references (moref)
export type VmwareDatacenter = {
id: string
name: string
}
export type VmwareCluster = {
id: string
name: string
datacenter?: string
}
export type VmwareHost = {
id: string
name: string
/** a cluster, or the compute resource of a standalone host */
parent?: string
datacenter?: string
}
export type VmwareDatastore = {
id: string
name?: string
type?: string
capacity?: number
freeSpace?: number
/** the hosts which mount the datastore and can reach it */
hosts: string[]
}
export type VmwareVlan = number | { start: number; end: number }[]
export type VmwareStandardNetwork = {
id: string
name: string
distributed: false
}
export type VmwareDistributedPortgroup = {
id: string
name: string
distributed: true
vlan?: VmwareVlan
}
export type VmwareNetwork = VmwareStandardNetwork | VmwareDistributedPortgroup
export type VmwareVmDisk = {
/** node of the disk, as the vmx names it, e.g. `scsi0:0` */
key: string
label: string
capacity: number
datastore?: string
diskMode?: string
rdm: boolean
encrypted: boolean
/** `dir/disk.vmdk` in its datastore of the base disk, then of each snapshot delta down to the active disk */
chainPaths: string[]
}
export type VmwareVmNic = {
/** device key of the card */
key: string
label: string
macAddress?: string
addressType?: string
network?: string
}
export type VmwareVm = {
id: string
name?: string
annotation?: string
instanceUuid?: string
firmware?: string
secureBoot: boolean
isTemplate: boolean
cpus?: number
memory?: number
powerState?: 'poweredOff' | 'poweredOn' | 'suspended'
host?: string
datastores: string[]
currentSnapshot?: string
snapshotCount: number
disks: VmwareVmDisk[]
nics: VmwareVmNic[]
}
export type VmwareInventory = {
/** e.g. `VMware vCenter Server`, `VMware ESXi` */
product?: string
version?: string
/** only set by a vCenter */
instanceUuid?: string
datacenters: VmwareDatacenter[]
clusters: VmwareCluster[]
hosts: VmwareHost[]
datastores: VmwareDatastore[]
networks: VmwareNetwork[]
vms: VmwareVm[]
}

View File

@@ -130,8 +130,6 @@ export type XsPatches = {
uuid?: string
}
type WrappedRecordOfType<Type extends WrappedXenApiRecord['$type']> = Extract<WrappedXenApiRecord, { $type: Type }>
export interface Xapi {
status: string
pool?: { uuid: string }
@@ -149,14 +147,6 @@ export interface Xapi {
ref: T['$ref'],
field: K
): Promise<T[K]>
// the record is inferred from `type`, for a JS caller which cannot spell it out as a type argument
getField<Type extends WrappedXenApiRecord['$type'], Field extends keyof WrappedRecordOfType<Type>>(
type: Type,
ref: WrappedRecordOfType<Type>['$ref'],
field: Field
): Promise<WrappedRecordOfType<Type>[Field]>
/** Synchronous: the record as the object cache currently holds it. */
getObjectByRef<Ref extends WrappedXenApiRecord['$ref']>(ref: Ref): Extract<WrappedXenApiRecord, { $ref: Ref }>
getObject: <
XoRecord extends XapiXoRecord,
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
@@ -293,42 +283,6 @@ export interface Xapi {
}
): Promise<XenApiVdi['$ref']>
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
VBD_create(params: {
bootable?: boolean
currently_attached?: boolean
device?: string
empty?: boolean
mode?: VBD_MODE
other_config?: Record<string, string>
qos_algorithm_params?: Record<string, string>
qos_algorithm_type?: string
/** a plug failure, only logged otherwise, is thrown: the VBD exists all the same */
throwVbdPlug?: boolean
type?: VBD_TYPE
unpluggable?: boolean
userdevice?: string
VDI?: XenApiVdi['$ref']
VM: XenApiVm['$ref']
}): Promise<XenApiVbd['$ref']>
/** unplugs the VBD first, if needed */
VBD_destroy(ref: XenApiVbd['$ref']): Promise<void>
VDI_create(
params: {
name_description?: string
name_label: string
other_config?: Record<string, string>
read_only?: boolean
sharable?: boolean
SR?: XenApiSr['$ref']
tags?: string[]
type?: VDI_TYPE
virtual_size: number
xenstore_data?: Record<string, string>
},
opts?: { sm_config?: Record<string, string> }
): Promise<XenApiVdi['$ref']>
/** a VDI which no longer exists counts as destroyed */
VDI_destroy(ref: XenApiVdi['$ref']): Promise<void>
editVm(
id: XoVm['id'],
props: EditVmProps,

View File

@@ -824,10 +824,7 @@ export interface XenApiSr {
VDIs: XenApiVdi['$ref'][]
virtual_allocation: number
}
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR', XenApiSrCallMethods> & {
// resolved by xen-api from `PBDs`
$PBDs: XenApiPbdWrapped[]
}
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR', XenApiSrCallMethods>
export interface XenApiSrStat {
$ref: Branded<'sr_stat'>

View File

@@ -1,37 +1,27 @@
import type { EventEmitter, Readable } from 'node:stream'
import { EventEmitter } from 'node:stream'
import type {
AnyXoJob,
AnyXoLog,
NonXapiXoRecord,
XoBackupDiskPartition,
XapiXoRecord,
XoAuthenticationToken,
XoBackupRepository,
XoConfigBackupArchive,
XoGroup,
XoHost,
XoMetadataBackupJob,
XoMirrorBackupJob,
XoNetwork,
XoPlugin,
XoPool,
XoPoolBackupArchive,
XoProxy,
XoSchedule,
XoServer,
XoSr,
XoTask,
XoUser,
XoVif,
XoVm,
XoVmBackupArchive,
XoVmBackupJob,
XoVmTemplate,
} from './xo.mjs'
import { VatesTask } from './lib/vates-task.mjs'
import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
import type { VmwareInventory } from './lib/xen-orchestra-vmware-explorer.mjs'
import type { XoV2vStatus } from './lib/xen-orchestra-v2v.mjs'
import type { RPU_RECOVERY_STEP_NAME } from './common.mjs'
import {
Xapi,
@@ -185,13 +175,13 @@ export type PoolRollingUpdateRecoveryBlocked = {
export type PoolRollingUpdateRecovery = PoolRollingUpdateRecoveryRun | PoolRollingUpdateRecoveryBlocked
/** A disk of a backup archive currently served as an iSCSI LUN */
/** A disk of a backup archive currently served as a read-only iSCSI LUN */
export type BackupArchiveDiskMount = {
/** Handle to pass to `unmountBackupArchiveDisk` */
id: string
/** UUID of the SR introduced on the host */
srUuid: string
/** UUID of the VDI exposing the backup disk, read-only unless it is cached */
/** UUID of the read-only VDI exposing the backup disk */
vdiUuid: string
/** IQN of the target serving the disk */
iqn: string
@@ -201,25 +191,7 @@ export type BackupArchiveDiskMount = {
port: number
}
type XoStoreRangeOptions = {
gt?: string
gte?: string
lt?: string
lte?: string
limit?: number
reverse?: boolean
}
// `@xen-orchestra/vmware-explorer` is not typed
export type EsxiConnectionParams = {
/** `host` or `host:port` */
host: string
user: string
password: string
sslVerify: boolean
}
export type XoApp = EventEmitter & {
export type XoApp = {
hooks: EventEmitter
_redis: {
get(key: string): Promise<string | null>
@@ -274,7 +246,6 @@ export type XoApp = EventEmitter & {
user?: XoUser
permission?: XoUser['permission'] | 'none' | null
}
backupLogsEe: EventEmitter
// methods ------------
allocIpAddresses(vifId: XoVif['id'], addAddresses?: string[], removeAddresses?: string[]): Promise<void>
@@ -299,8 +270,6 @@ export type XoApp = EventEmitter & {
backupGuard(objectId: XapiXoRecord['id'], opts?: { bypassBackupCheck?: boolean; operation: string }): Promise<void>
/* Throw if no authorization */
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
/* whether this machine can run the tools of an import from VMware */
checkEsxiDependencies(): Promise<XoV2vStatus>
/* validate, apply and persist the configuration of a plugin */
configurePlugin(id: string, configuration: unknown, mergeWithExisting?: boolean): Promise<void>
/* connect a server (XCP-ng/XenServer) */
@@ -316,29 +285,7 @@ export type XoApp = EventEmitter & {
id: XoAclRole['id'],
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
): Promise<XoAclRole['id']>
createAclV2Role(role: {
name: XoAclRole['name']
description?: XoAclRole['description']
privileges?: Omit<XoAclBasePrivilege, 'id' | 'roleId'>[]
}): Promise<XoAclRole>
createApiConnection(remoteAddress: string): EventEmitter & {
close(): void
get<T = unknown>(key: string, defaultValue: unknown): T
has(key: string): boolean
set(key: string, value: unknown): void
unset(key: string): void
}
createBackupNgJob<T extends XoVmBackupJob | XoMirrorBackupJob>(
type: T['type'],
props: Omit<T, 'id' | 'type'>,
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
schedules?: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
): Promise<T>
createMetadataBackupJob(
props: Omit<XoMetadataBackupJob, 'id' | 'type'>,
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
schedules: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
): Promise<XoMetadataBackupJob>
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
createAuthenticationToken(opts: {
client?: {
id?: string
@@ -354,14 +301,6 @@ export type XoApp = EventEmitter & {
proxy?: XoProxy['id']
url: string
}): Promise<XoBackupRepository>
createSchedule(params: {
cron: XoSchedule['cron']
enabled: XoSchedule['enabled']
jobId: XoSchedule['jobId']
name?: XoSchedule['name']
timezone?: XoSchedule['timezone']
userId?: XoUser['id']
}): Promise<XoSchedule>
createUser(params: {
firstname?: string
lastname?: string
@@ -378,14 +317,11 @@ export type XoApp = EventEmitter & {
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteSchedule(id: XoSchedule['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
detachHostFromPool(hostId: XoHost['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
disablePluginAutoload(id: XoPlugin['id']): Promise<void>
/* disconnect a server (XCP-ng/XenServer) */
disconnectXenServer(id: XoServer['id']): Promise<void>
enablePluginAutoload(id: XoPlugin['id']): Promise<void>
findEnabledScheduleSequenceFromSchedule(id: XoSchedule['id']): Promise<XoSchedule | undefined>
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
@@ -396,7 +332,6 @@ export type XoApp = EventEmitter & {
roleId: XoAclRole['id'],
opts?: { bypassAuthorization?: boolean }
): Promise<boolean>
deleteSchedule(id: XoSchedule['id']): Promise<void>
getAclV2GroupRoles(
groupId: XoGroup['id'],
opts?: { bypassAuthorization?: boolean }
@@ -411,14 +346,10 @@ export type XoApp = EventEmitter & {
getAllProxies(): Promise<XoProxy[]>
getAllJobs<T extends AnyXoJob['type']>(type: T): Promise<Extract<AnyXoJob, { type: T }>[]>
getAllJobs(type?: string): Promise<AnyXoJob[]>
getPlugin(id: XoPlugin['id']): Promise<XoPlugin>
getPlugins(): Promise<XoPlugin[]>
getProxy(id: XoProxy['id']): Promise<XoProxy>
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
getAllRemotes(): Promise<XoBackupRepository[]>
removeRemote(id: XoBackupRepository['id']): Promise<void>
/** Also deletes the schedules of the job */
removeJob(id: AnyXoJob['id']): Promise<void>
getAllRemotesInfo(): Promise<
Record<
XoBackupRepository['id'],
@@ -447,10 +378,6 @@ export type XoApp = EventEmitter & {
}
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
/** Please use `REMOVE_CACHE_ENTRY` const to clear the cache */
getBackupNgLogs(removeCacheEntry: {}): void
/** Please use `REMOVE_CACHE_ENTRY` const to clear a cache entry */
getBackupNgLogs(removeCacheEntry: {}, id: AnyXoLog['id']): void
getBackupNgLogsSorted(opts: {
after?: number
before?: number
@@ -468,21 +395,8 @@ export type XoApp = EventEmitter & {
) => Record<T['id'], T> | undefined
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
getStore(namespace: string): Promise<{
/** rejects with an error having `notFound: true` when the key does not exist */
get(key: string): Promise<unknown>
has(key: string): Promise<boolean>
put(key: string, value: unknown): Promise<void>
del(key: string): Promise<void>
del(key: string, cb: (error?: Error | null) => void): void
/** object mode stream of `{ key, value }` */
createReadStream(opts?: XoStoreRangeOptions): Readable
/** object mode stream of keys */
createKeyStream(opts?: XoStoreRangeOptions): Readable
}>
getUser: (id: XoUser['id'], opts?: { obfuscatePassword?: boolean }) => Promise<XoUser>
getUserIdentityFields(): string[]
getEsxiInventory(params: EsxiConnectionParams): Promise<VmwareInventory>
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
getXapiObject: <T extends XapiXoRecord>(
@@ -492,29 +406,9 @@ export type XoApp = EventEmitter & {
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
getXenServer(id: XoServer['id']): Promise<XoServer>
/** the connection of the pool holding this object, which may not be connected */
getXenServerIdByObject<T extends XapiXoRecord>(maybeId: T['id'] | T, type?: T['type'] | T['type'][]): XoServer['id']
getXenServerWithCredentials(id: XoServer['id']): Promise<XoServer & { password: string }>
getXoEventEmitterByType(type: string): EventEmitter
hasFeatureAuthorization(featureCode: string): Promise<boolean>
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
listBackupNgDiskPartitions(remoteId: XoBackupRepository['id'], diskId: string): Promise<XoBackupDiskPartition[]>
listBackupNgPartitionFiles(
remoteId: XoBackupRepository['id'],
diskId: string,
partitionId: string | undefined,
path: string
// keys ending in "/" are directories; files carry size (bytes) and mtime (epoch ms).
// Values may be empty ({}) for directories or when served by an older proxy.
): Promise<Record<string, { size?: number; mtime?: number }>>
fetchBackupNgPartitionFiles(
remoteId: XoBackupRepository['id'],
diskId: string,
partitionId: string | undefined,
paths: string[],
format: string
): Promise<Readable>
loadPlugin(id: XoPlugin['id']): Promise<void>
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
@@ -524,41 +418,19 @@ export type XoApp = EventEmitter & {
backupRepositoryIds: XoBackupRepository['id'][],
opts?: { _forceRefresh?: boolean; vmId?: XoVm['id'] }
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]> | null>>
/** imports one VM from VMware, resuming a previous partial import when possible, returns the UUID of the XCP-ng VM */
migrationfromEsxi(
params: EsxiConnectionParams & {
sr: XoSr['id']
network: XoNetwork['id']
/** overrides `sr`, by disk node (e.g. `scsi0:0`) */
disks?: Record<string, XoSr['id']>
/** overrides `network`, by NIC device key (e.g. `4000`) */
nics?: Record<string, XoNetwork['id']>
template: XoVmTemplate['id']
stopSource: boolean
/** VMware moref of the VM */
vm: string
}
): Promise<XoVm['uuid']>
/**
* Serve one disk of a backup archive as an iSCSI LUN and attach it to `host`
* as an SR, read-only unless `cacheSrId` is set. Undone by `unmountBackupArchiveDisk`.
* Serve one disk of a backup archive as a read-only iSCSI LUN and attach it to
* `host` as an SR. Undone by `unmountBackupArchiveDisk`.
*
* The LUN is served by whoever can read the backup repository: this appliance, or the proxy the
* repository is linked to.
*/
mountBackupArchiveDisk(params: {
archiveId: XoVmBackupArchive['id']
/**
* SR of a VDI of the serving appliance, which the disk is materialized into as it is read and
* which holds the writes; unset, nothing is cached and the mount is read-only. On a local SR,
* the serving appliance must not be migrated while the mount lasts: the VDI cannot follow it
*/
cacheSrId?: XoSr['id']
/** One of the archive's `disks[].id` */
diskId: string
hostId: XoHost['id']
}): Promise<BackupArchiveDiskMount>
purgePluginConfiguration(id: XoPlugin['id']): Promise<void>
pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
/**
* Record the live mounts a proxy created by itself, while running a restore: they never went
@@ -599,7 +471,6 @@ export type XoApp = EventEmitter & {
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
runJob(job: AnyXoJob, schedule: XoSchedule): void
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
testPlugin(id: XoPlugin['id'], data?: unknown): Promise<void>
testRemote(
id: XoBackupRepository['id']
): Promise<
@@ -609,26 +480,12 @@ export type XoApp = EventEmitter & {
touchXoObject(type: string, id: NonXapiXoRecord['id']): Promise<void>
/** Detach a disk mounted by `mountBackupArchiveDisk` and stop serving it */
unmountBackupArchiveDisk(id: BackupArchiveDiskMount['id']): Promise<void>
unloadPlugin(id: XoPlugin['id']): Promise<void>
reclaimSpace(
remoteId: XoBackupRepository['id'],
opts?: { vmUuid?: XoVm['id']; merge?: boolean; remove?: boolean }
): Promise<{ vmUuid: string; success: boolean; merge?: boolean; size?: number; error?: string }[]>
/** Remove a server from the DB (XCP-ng/XenServer) */
unregisterXenServer(id: XoServer['id']): Promise<void>
/**
* Shallow merge of the properties into the existing job, a `null` value removes the property.
*/
updateJob(job: { id: AnyXoJob['id']; [key: string]: unknown }, merge?: boolean): Promise<void>
updateSchedule(params: {
id: XoSchedule['id']
cron?: XoSchedule['cron']
enabled?: XoSchedule['enabled']
jobId?: XoSchedule['jobId']
name?: XoSchedule['name']
timezone?: XoSchedule['timezone']
userId?: XoUser['id']
}): Promise<void>
updateUser(
id: XoUser['id'],
updates: {

View File

@@ -21,29 +21,6 @@ import type {
import type * as CMType from './lib/complex-matcher.mjs'
import { XoAclBasePrivilege, XoAclRole } from './lib/xen-orchestra-acl.mjs'
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
import type {
XoAuditPlugin,
XoAuthGithubPlugin,
XoAuthGooglePlugin,
XoAuthLdapPlugin,
XoAuthOidcPlugin,
XoAuthSamlPlugin,
XoBackupReportsPlugin,
XoIpmiSensorsPlugin,
XoLoadBalancerPlugin,
XoNetboxPlugin,
XoOpenmetricsPlugin,
XoPerfAlertPlugin,
XoSdnControllerPlugin,
XoTestPluginPlugin,
XoTransportEmailPlugin,
XoTransportIcinga2Plugin,
XoTransportNagiosPlugin,
XoTransportSlackPlugin,
XoTransportXmppPlugin,
XoUsageReportPlugin,
XoWebHooksPlugin,
} from './lib/xen-orchestra-plugins.mjs'
type BaseXapiXo = {
$pool: XoPool['id']
@@ -141,7 +118,7 @@ type BaseXoVm = BaseXapiXo & {
/**
* @deprecated use pvDriversVersion instead
*/
xenTools?:
xentools?:
| false
| {
major: null | number
@@ -180,15 +157,6 @@ export type XoBackupLog = BaseXoLog & {
export type XoRestoreLog = BaseXoLog & {
message: 'restore'
}
export type XoBackupDiskPartition = {
id: Branded<'xo-backup-disk-partition'>
name?: string
size: number
start: number
/** number for MBR, string for GPT partition */
type: string | number
}
export type XoVmBackupArchive = {
id: Branded<'vm-backup-archive'>
type: 'xo-vm-backup'
@@ -240,7 +208,6 @@ export type XoBackupRepositoryBenchmark = {
}
export type XoBackupRepository = {
available?: number
benchmarks?: XoBackupRepositoryBenchmark[]
enabled: boolean
error?: Record<string, unknown>
@@ -248,9 +215,7 @@ export type XoBackupRepository = {
name: string
options?: string
proxy?: XoProxy['id']
size?: number
url: string
used?: number
}
export type XoGpuGroup = BaseXapiXo & {
@@ -492,29 +457,6 @@ export type XoPool = BaseXapiXo & {
zstdSupported: boolean
}
export type XoPlugin =
| XoAuditPlugin
| XoAuthGithubPlugin
| XoAuthGooglePlugin
| XoAuthLdapPlugin
| XoAuthOidcPlugin
| XoAuthSamlPlugin
| XoBackupReportsPlugin
| XoIpmiSensorsPlugin
| XoLoadBalancerPlugin
| XoNetboxPlugin
| XoOpenmetricsPlugin
| XoPerfAlertPlugin
| XoSdnControllerPlugin
| XoTestPluginPlugin
| XoTransportEmailPlugin
| XoTransportIcinga2Plugin
| XoTransportNagiosPlugin
| XoTransportSlackPlugin
| XoTransportXmppPlugin
| XoUsageReportPlugin
| XoWebHooksPlugin
export type XoProxy = {
id: Branded<'proxy'>
url: string
@@ -539,22 +481,15 @@ type XoBackupJobGeneralSettings = {
[key: string]: unknown
}
/**
* Keeps the last backup of each of the last `retention` days/weeks/months/years, in addition to `exportRetention`.
*
* Requires the `timezone` setting.
*/
export type XoBackupJobLongTermRetention = {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
cbtDestroySnapshotData?: boolean
concurrency?: number
longTermRetention?: XoBackupJobLongTermRetention
longTermRetention?: {
daily?: { retention: number; settings: Record<string, unknown> }
weekly?: { retention: number; settings: Record<string, unknown> }
monthly?: { retention: number; settings: Record<string, unknown> }
yearly?: { retention: number; settings: Record<string, unknown> }
}
checkpointSnapshot?: boolean
offlineSnapshot?: boolean
maxExportRate?: number
@@ -568,8 +503,8 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
}
export type XoVmBackupJobScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags'] | string
fullInterval?: number | string
healthCheckVmsWithTags?: XoVm['tags']
fullInterval?: number
copyRetention?: number
snapshotRetention?: number
cbtDestroySnapshotData?: boolean
@@ -580,9 +515,9 @@ export type XoVmBackupJob = BaseXoJob & {
compression?: 'native' | 'zstd' | ''
proxy?: XoProxy['id']
mode: 'full' | 'delta'
remotes?: CMType.IdOr<XoBackupRepository['id']> | string
remotes?: CMType.IdOr<XoBackupRepository['id']>
vms: CMType.IdOr<XoVm['id']> | Record<string, unknown>
srs?: CMType.IdOr<XoSr['id']> | string
srs?: CMType.IdOr<XoSr['id']>
type: 'backup'
settings: {
'': XoVmBackupJobGeneralSettings
@@ -611,17 +546,16 @@ export type XoMetadataBackupJob = BaseXoJob & {
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
concurrency?: number
longTermRetention?: XoBackupJobLongTermRetention
nRetriesVmBackupFailures?: number
mergeBackupsSynchronously?: boolean
timeout?: number
timezone?: string
maxExportRate?: number
backupReportTpl?: 'compactMjml'
reportWhen: 'failure'
}
export type XoMirrorBackupScheduleSettings = {
exportRetention?: number
healthCheckVmsWithTags?: XoVm['tags'] | string
vmFilter?: string
healthCheckVmsWithTags?: XoVm['tags']
healthCheckSr?: XoSr['id']
[key: string]: unknown
}
@@ -675,7 +609,6 @@ export type XoServer = {
label?: string
master?: XoHost['id']
poolId?: XoPool['id']
poolMembersAddresses?: string[]
poolNameDescription?: string
poolNameLabel?: string
readOnly: boolean
@@ -695,7 +628,6 @@ export type XoSr = BaseXapiXo & {
current_operations: Record<string, STORAGE_OPERATIONS>
id: Branded<'SR'>
inMaintenanceMode: boolean
is_tools_sr: boolean
name_description: string
name_label: string
other_config: Record<string, string>
@@ -764,7 +696,6 @@ export type XoTask = {
| 'backup-log'
| 'backup-repository'
| 'group'
| 'plugin'
| 'proxy'
| 'restore'
| 'restore-log'
@@ -978,9 +909,7 @@ export type NonXapiXoRecord =
| AnyXoBackupArchive
| AnyXoJob
| AnyXoLog
| XoAuthenticationToken
| XoGroup
| XoPlugin
| XoProxy
| XoJob
| XoBackupRepository

View File

@@ -6,7 +6,7 @@
"main": "dist/index.mjs",
"name": "@xen-orchestra/acl",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
"version": "1.8.0",
"version": "1.7.0",
"license": "AGPL-3.0-or-later",
"private": false,
"type": "module",
@@ -30,7 +30,7 @@
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"@vates/types": "^2.5.0",
"@vates/types": "^2.3.0",
"rimraf": "^6.0.1",
"typescript": "~5.6",
"typescript-eslint": "^8.61.0"

View File

@@ -1,5 +1,3 @@
export default {
export: true, // can export the content ( full or file by file )
read: true, // can read the metadata of an archive
mount: true, // can mount the archive for file level restore
read: true,
}

View File

@@ -25,14 +25,4 @@ export default {
update: {
tags: true,
},
probe: {
nfs: true,
zfs: true,
hba: true,
iscsiiqn: true,
iscsilun: true,
'iscsi-exists': true,
'hba-exists': true,
'nfs-exists': true,
},
}

View File

@@ -14,7 +14,6 @@ import { default as pbd } from './pbd.mjs'
import { default as pci } from './pci.mjs'
import { default as pgpu } from './pgpu.mjs'
import { default as pif } from './pif.mjs'
import { default as plugin } from './plugin.mjs'
import { default as pool } from './pool.mjs'
import { default as proxy } from './proxy.mjs'
import { default as restoreLog } from './restore-log.mjs'
@@ -57,7 +56,6 @@ export const SUPPORTED_ACTIONS_BY_RESOURCE = {
pci,
pgpu,
pif,
plugin,
pool,
proxy,
'restore-log': restoreLog,

View File

@@ -1,10 +0,0 @@
export default {
load: true,
read: true,
test: true,
unload: true,
update: {
autoload: true,
configuration: true,
},
}

View File

@@ -1,3 +1,4 @@
import assert from 'node:assert'
import type { XoUser } from '@vates/types/xo'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
@@ -41,21 +42,10 @@ export function hasPrivilegeOn<T extends SupportedResource>({
}) {
// Function that will be called outside of the module
// We cannot be sure types are respected
if (typeof user?.permission !== 'string') {
throw new TypeError('user.permission must be a string')
}
if (typeof action !== 'string') {
throw new TypeError('action must be a string')
}
if (typeof resource !== 'string') {
throw new TypeError('resource must be a string')
}
if (objects === undefined) {
throw new TypeError('objects must be defined')
}
assert.strictEqual(typeof user?.permission, 'string')
assert.strictEqual(typeof action, 'string')
assert.strictEqual(typeof resource, 'string')
assert.strictEqual(objects === undefined, false)
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)

View File

@@ -1,6 +1,6 @@
{
"name": "@xen-orchestra/backup-archive",
"version": "1.2.0",
"version": "1.1.3",
"description": "Helper to handle generic backup archives",
"license": "ISC",
"author": "",
@@ -21,25 +21,25 @@
"tar-stream": "^3.1.8",
"@types/tar-stream": "^3.1.4",
"tmp": "^0.2.1",
"@vates/types": "^2.5.0"
"@vates/types": "^2.3.0"
},
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/fs": "^4.10.0",
"@xen-orchestra/disk-transform": "^1.3.4",
"@xen-orchestra/fs": "^4.9.5",
"@xen-orchestra/disk-transform": "^1.3.3",
"@vates/task": "0.7.0",
"golike-defer": "0.5.1",
"uuid": "^9.0.0",
"promise-toolbox": "^0.21.0",
"@vates/async-each": "^1.0.1",
"vhd-lib": "^4.17.0"
"vhd-lib": "^4.16.2"
},
"scripts": {
"build": "yarn run clean && tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "yarn run clean && tsc && node --test './dist/**/*.test.mjs'",
"test-integration": "tsc && node --test './src/**/*.integ.mjs'",
"test": "yarn run clean && tsc && node --test **/*.test.mjs",
"test-integration": "tsc && node --test **/*.integ.mjs",
"clean": "rimraf dist/"
},
"private": false,

View File

@@ -42,13 +42,12 @@ export class RemoteDiskLineage {
#childOf: Map<string, string> = new Map()
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
#activeDiskPaths: Set<string> = new Set()
// Interrupted merges: normalized parent path { stateFilePath, chain, step }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[]; step?: 'mergeBlocks' | 'cleanup' }> =
new Map()
// Interrupted merges: normalized parent path { stateFilePath, chain }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
this.#handler = handler
this.#vdiDir = normalize(vdiDir)
this.#vdiDir = vdiDir
this.#opts = opts
}
@@ -267,38 +266,8 @@ export class RemoteDiskLineage {
return undefined
}
// Read-only walk toward the descendants: true if any of them is still referenced by a backup.
// Unlike getUsedChildChainOrDelete, it neither marks disks visited nor schedules deletions.
const hasUsedDescendant = (diskPath: string): boolean => {
// `seen` guards against a cycle in corrupted headers
const seen = new Set<string>()
for (let p: string | undefined = diskPath; p !== undefined && !seen.has(p); p = this.#childOf.get(p)) {
seen.add(p)
if (!orphanDisks.has(p)) {
return true
}
}
return false
}
// Keeps the orphan loop away from a chain that must not be touched this run. Pinned disks are
// differencing: every ancestor they read through must survive too, however deep the lineage is
// (e.g. after a retention change). The ancestors are merged normally once the chain is
// resumed or dropped.
const pin = (paths: readonly string[]): void => {
for (const path of paths) {
// `seen` guards against a cycle in corrupted headers
const seen = new Set<string>()
for (let p: string | undefined = path; p !== undefined && !seen.has(p); p = this.#parentOf.get(p)) {
seen.add(p)
visited.add(p)
toDelete.delete(p)
}
}
}
// Process interrupted merges first so their disks are protected from the orphan loop
for (const [parentPath, { stateFilePath, chain: stateChain, step }] of this.#interruptedMerges) {
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
if (!this.#diskPaths.has(parentPath)) {
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
if (remove) {
@@ -310,85 +279,14 @@ export class RemoteDiskLineage {
let chain: string[] | undefined
if (stateChain !== undefined) {
const missing = stateChain.filter(p => !this.#diskPaths.has(p))
// A merge only removes disks during its cleanup step, so outside of it every disk of the
// recorded chain must still be there. Merging what is left would fold the surviving
// children into the parent and then rename it onto the chain tip, silently dropping the
// blocks of the disks that went missing. Never resume such a chain.
if (missing.length > 0 && step !== 'cleanup') {
// Each surviving disk is walked on its own: the missing disk breaks the `childOf` links,
// so a walk from the parent alone would stop at the gap.
const survivors = stateChain.filter(p => this.#diskPaths.has(p))
if (survivors.some(hasUsedDescendant)) {
// A backup still references a disk downstream: pin the chain so the orphan loop
// neither merges it afresh nor deletes it. It is dropped once nothing uses it anymore.
this.#opts.logWarn('merge chain lost disks before its cleanup step, refusing to resume', {
stateFilePath,
parentPath,
step,
missing,
})
pin(survivors)
continue
}
// Nothing uses the chain anymore (descendants of the missing disk are broken, so never
// active): drop the state, the orphan loop below deletes the disks.
this.#opts.logWarn('unmergeable merge chain on a fully orphaned lineage', {
stateFilePath,
parentPath,
step,
missing,
})
if (remove) {
this.#opts.logInfo('deleting merge state of orphaned lineage', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
const existing = stateChain.filter(p => this.#diskPaths.has(p))
if (existing.length >= 2) {
chain = existing
} else {
// fewer than 2 disks of the recorded chain are left, there is nothing to resume and
// the state would otherwise stay on disk forever.
//
// only done when the chain was read successfully: an undefined `stateChain` also
// covers a state file that is merely unreadable right now, which must be kept
this.#opts.logWarn('merge state without a resumable chain', { stateFilePath, parentPath })
if (remove) {
this.#opts.logInfo('deleting unresumable merge state', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
if (existing.length >= 2) chain = existing
}
if (chain === undefined) {
// state file unreadable: leave it alone, the orphan loop below schedules a fresh merge
continue
if (chain !== undefined) {
chain.forEach(p => visited.add(p))
toMerge.push({ chain, isResuming: true })
}
// Nothing downstream of the merge parent is still referenced by a backup: merging would
// write into a lineage nothing points at, and pinning it below would exempt it from the
// orphan loop forever. The walk already marked the whole lineage for deletion, so only
// the state file is left to drop.
//
// It walks the live lineage rather than `stateChain`, which is frozen at the retention
// boundary of the run that created it and can be shorter than the real chain.
if (getUsedChildChainOrDelete(parentPath) === undefined) {
this.#opts.logWarn('merge state on a fully orphaned lineage', { stateFilePath, parentPath })
if (remove) {
this.#opts.logInfo('deleting merge state of orphaned lineage', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
pin(chain)
toMerge.push({ chain, isResuming: true })
}
for (const orphan of orphanDisks) {

View File

@@ -1,279 +0,0 @@
import { test } from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { normalize } from '@xen-orchestra/fs/path'
import { pFromCallback } from 'promise-toolbox'
import { rimraf } from 'rimraf'
import { Constants, VhdAbstract, VhdDirectory } from 'vhd-lib'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
// `RemoteDiskLineage` is a `.mts` module: this is the TS ESM spelling of the sibling import, which
// resolves at runtime because the `test` script runs the built `dist/` copy of this file. It must
// NOT go through `../dist/`, which would make `tsc` pull the emitted declarations back in as
// inputs (TS5055) — the `*.integ.mjs` files can only do that because tsconfig excludes them.
/* eslint-disable n/no-missing-import */
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
/* eslint-enable n/no-missing-import */
const { beforeEach, afterEach, describe } = test
let handler, tempDir, vdiDir
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
// VmIncrementalBackupArchive passes a normalized path, and RemoteDiskLineage compares
// #vdiDir against normalized paths, so the fixture has to match
vdiDir = normalize(`xo-vm-backups/VMUUID/vdis/${uuid.v1()}/${uuid.v1()}`)
await handler.mktree(`${vdiDir}/data`)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
// Creates an aliased VHD directory in `vdiDir`, chained onto `parent` when given.
async function generateDisk(name, { parent } = {}) {
const aliasPath = `${vdiDir}/${name}.alias.vhd`
const dataPath = `${vdiDir}/data/${name}.vhd`
await handler.mkdir(dataPath)
const vhd = new VhdDirectory(handler, dataPath)
vhd.header = { ...VHDHEADER }
vhd.footer = { ...VHDFOOTER, uuid: uuid.v1({}, Buffer.alloc(16)) }
if (parent !== undefined) {
vhd.header.parentUnicodeName = `${parent.name}.alias.vhd`
vhd.header.parentUuid = parent.vhd.footer.uuid
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
await vhd.writeEntireBlock({ id: 0, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, 1) })
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
await VhdAbstract.createAlias(handler, aliasPath, dataPath)
return { name, aliasPath, dataPath, vhd }
}
// Plants the merge state file a previous interrupted merge would have left on `parent`.
async function writeMergeState(parent, { chain, step = 'mergeBlocks' }) {
const statePath = `${vdiDir}/.${parent.name}.alias.vhd.merge.json`
await handler.writeFile(
statePath,
JSON.stringify({
child: { uuid: uuid.v1() },
parent: { uuid: uuid.v1() },
chain: chain.map(name => `${name}.alias.vhd`),
currentBlock: 0,
mergedDataSize: 0,
step,
diskSize: 0,
})
)
return statePath
}
async function cleanLineage({ activeDisks = [], lineageDir = vdiDir, merge = false } = {}) {
const warnings = []
const infos = []
const lineage = new RemoteDiskLineage(handler, lineageDir, {
remove: true,
merge,
logInfo: (message, data) => infos.push({ message, data }),
logWarn: (message, data) => warnings.push({ message, data }),
})
await lineage.init()
for (const disk of activeDisks) {
lineage.addActiveDiskPath(disk.aliasPath)
}
await lineage.clean({ remove: true, merge })
return { warnings, infos }
}
const exists = path => handler.list(vdiDir).then(files => files.includes(path.slice(vdiDir.length + 1)))
describe('RemoteDiskLineage.clean() interrupted merges', { concurrency: 1 }, () => {
test('drops the merge state and the disks when no descendant is still referenced', async () => {
const base = await generateDisk('base')
const child = await generateDisk('child', { parent: base })
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
// no active disk: the whole lineage is out of retention
const { warnings } = await cleanLineage()
assert.equal(await exists(statePath), false, 'merge state should be deleted')
assert.equal(await exists(base.aliasPath), false, 'base should be deleted')
assert.equal(await exists(child.aliasPath), false, 'child should be deleted')
assert.ok(
warnings.some(({ message }) => message === 'merge state on a fully orphaned lineage'),
'the dropped chain should be reported'
)
})
test('refuses to resume, and keeps everything, when the chain lost a disk before its cleanup step', async () => {
const base = await generateDisk('base')
const child = await generateDisk('child', { parent: base })
// `missing` was recorded in the chain but is no longer on the remote: merging base + child
// and renaming onto the tip would silently drop its blocks
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
// merge enabled: the orphan loop must not pick the refused chain up and merge it afresh
const { warnings, infos } = await cleanLineage({ activeDisks: [child], merge: true })
assert.equal(
infos.some(({ message }) => message === 'Disk chain needs merging'),
false,
'no merge should be scheduled for a chain that was refused'
)
assert.equal(await exists(statePath), true, 'merge state should be kept while a backup uses the chain')
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
const warning = warnings.find(
({ message }) => message === 'merge chain lost disks before its cleanup step, refusing to resume'
)
assert.notEqual(warning, undefined, 'the incomplete chain should be reported')
assert.deepEqual(warning.data.missing, [`${vdiDir}/missing.alias.vhd`])
})
test('keeps every ancestor of a chain it refused to resume', async () => {
// several ancestors fell out of retention after the merge was interrupted (e.g. the retention
// was lowered): deleting any of them would leave `base`, a differencing disk, without a parent
const ggp = await generateDisk('ggp')
const gp = await generateDisk('gp', { parent: ggp })
const base = await generateDisk('base', { parent: gp })
const child = await generateDisk('child', { parent: base })
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
const { warnings, infos } = await cleanLineage({ activeDisks: [child], merge: true })
assert.ok(
warnings.some(({ message }) => message === 'merge chain lost disks before its cleanup step, refusing to resume'),
'the incomplete chain should be reported'
)
assert.equal(
infos.some(({ message }) => message === 'Disk chain needs merging'),
false,
'neither the refused chain nor its ancestors should be merged'
)
assert.equal(await exists(ggp.aliasPath), true, 'ggp should be kept')
assert.equal(await exists(gp.aliasPath), true, 'gp should be kept')
assert.equal(await exists(statePath), true, 'merge state should be kept')
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
})
test('keeps every ancestor of a resumed chain, and merges them on the next run', async () => {
const ggp = await generateDisk('ggp')
const gp = await generateDisk('gp', { parent: ggp })
const base = await generateDisk('base', { parent: gp })
const child = await generateDisk('child', { parent: base })
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
// first run: only the interrupted merge is resumed, its ancestors are left alone
const firstRun = await cleanLineage({ activeDisks: [child], merge: true })
assert.ok(
firstRun.infos.some(({ message, data }) => message === 'merging disk chain' && data.chain.length === 2),
'the interrupted merge should be resumed'
)
assert.equal(await exists(statePath), false, 'merge state should be deleted once the merge completed')
assert.equal(await exists(base.aliasPath), false, 'base should be merged into child')
assert.equal(await exists(ggp.aliasPath), true, 'ggp should be kept')
assert.equal(await exists(gp.aliasPath), true, 'gp should be kept')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
// second run: the ancestors are merged into child as a regular chain
const secondRun = await cleanLineage({ activeDisks: [child], merge: true })
assert.deepEqual(
secondRun.warnings.filter(({ message }) => message === 'disk broken: parent missing or broken'),
[],
'child should still have its parents'
)
assert.ok(
secondRun.infos.some(({ message, data }) => message === 'merging disk chain' && data.chain.length === 3),
'ggp and gp should be merged into child'
)
assert.equal(await exists(ggp.aliasPath), false, 'ggp should be merged into child')
assert.equal(await exists(gp.aliasPath), false, 'gp should be merged into child')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
})
test('drops an unmergeable chain, state and disks, once nothing uses it', async () => {
const base = await generateDisk('base')
const missing = await generateDisk('missing', { parent: base })
const child = await generateDisk('child', { parent: missing })
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
// the intermediate disk vanished mid-merge: child is now broken, so its backup is incomplete
// and never declares it active
await handler.unlink(missing.aliasPath)
await handler.rmtree(missing.dataPath)
const { warnings, infos } = await cleanLineage({ merge: true })
assert.equal(
infos.some(({ message }) => message === 'Disk chain needs merging'),
false,
'an unmergeable chain should not be merged'
)
assert.equal(await exists(statePath), false, 'merge state should be deleted')
assert.equal(await exists(base.aliasPath), false, 'base should be deleted')
assert.equal(await exists(child.aliasPath), false, 'child should be deleted')
assert.ok(
warnings.some(({ message }) => message === 'unmergeable merge chain on a fully orphaned lineage'),
'the dropped chain should be reported'
)
})
test('drops a merge state whose chain no longer has two disks to merge, during cleanup', async () => {
const base = await generateDisk('base')
// the cleanup step unlinks the children as it goes, so a gap there is expected
const statePath = await writeMergeState(base, { chain: ['base', 'alreadyUnlinked'], step: 'cleanup' })
const { warnings } = await cleanLineage({ activeDisks: [base] })
assert.equal(await exists(statePath), false, 'merge state should be deleted')
assert.equal(await exists(base.aliasPath), true, 'the active disk should be kept')
assert.ok(
warnings.some(({ message }) => message === 'merge state without a resumable chain'),
'the unresumable state should be reported'
)
})
test('keeps a resumable merge state and its disks', async () => {
const base = await generateDisk('base')
const child = await generateDisk('child', { parent: base })
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
const { warnings } = await cleanLineage({ activeDisks: [child] })
assert.equal(await exists(statePath), true, 'merge state should be kept')
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
assert.deepEqual(warnings, [], 'a resumable merge should not warn')
})
test('behaves the same when built with a non-normalized VDI directory', async () => {
const base = await generateDisk('base')
const child = await generateDisk('child', { parent: base })
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
// the constructor must normalize: otherwise #cleanOrphanDataFiles fails to exclude the VDI
// directory from the data directories it sweeps and deletes every unclaimed file in it,
// starting with the merge state
const { warnings } = await cleanLineage({ activeDisks: [child], lineageDir: vdiDir.replace(/^\//, '') })
assert.equal(await exists(statePath), true, 'merge state should be kept')
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
assert.deepEqual(warnings, [], 'nothing should be reported as orphaned')
})
})

View File

@@ -331,7 +331,7 @@ export class MergeRemoteDisk {
* @param {string[]} files - all file paths in vdiDir (prependDir: true)
* @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
* @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[], step?: 'mergeBlocks' | 'cleanup' }>>}
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
*/
static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
@@ -343,12 +343,8 @@ export class MergeRemoteDisk {
const parentPath = normalize(vdiDir + '/' + match[1])
let chain
let step
try {
const state = JSON.parse(await handler.readFile(filePath))
// disks are only removed during the cleanup step, so the caller needs the step to tell
// an expected gap in the chain from a lineage that lost data
step = state?.step ?? 'mergeBlocks'
if (Array.isArray(state?.chain)) {
chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
} else {
@@ -361,7 +357,7 @@ export class MergeRemoteDisk {
warn("Merge state unreadable, can't restart merging.", error)
}
result.set(parentPath, { stateFilePath: filePath, chain, step })
result.set(parentPath, { stateFilePath: filePath, chain })
}
return result

View File

@@ -5,7 +5,6 @@
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
* @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
* @typedef {import('vhd-lib/disk-consumer/compressedBlock.mjs').CompressedDiskBlock} CompressedDiskBlock
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
*
@@ -46,36 +45,26 @@ export class RemoteVhdDisk extends RemoteDisk {
*/
#blockSize = 2 * 1024 * 1024
/**
* @type {number}
*/
#bitmapSize = 512
/**
* @type {() => any}
*/
#dispose = () => {}
/**
* @type {string | undefined}
*/
#rawBlocksCompression
/**
* set in init(): this disk is a VHD directory using #rawBlocksCompression
* @type {boolean}
*/
#rawBlocks = false
/**
* @param {Object} params
* @param {RemoteHandlerAbstract} params.handler
* @param {string} params.path
* @param {string} [params.rawBlocksCompression] if this disk is a VHD directory using this compression, its blocks
* are read as stored (see CompressedDiskBlock), to be written as is in a VHD directory using the same compression.
* Otherwise (plain VHD, other compression), they are read normally
*/
constructor({ handler, path, rawBlocksCompression }) {
constructor({ handler, path }) {
super()
// @todo : ensure this is the full path from the root of the remote
this.#path = path
this.#handler = handler
this.#rawBlocksCompression = rawBlocksCompression
}
/**
@@ -98,9 +87,6 @@ export class RemoteVhdDisk extends RemoteDisk {
this.#vhd = undefined
throw Object.assign(new Error("Can't init vhd directory without using alias"), { code: 'NOT_SUPPORTED' })
}
// the disk may predate a change of the remote configuration: only use its raw blocks if they match
this.#rawBlocks =
this.#rawBlocksCompression !== undefined && this.#getCompressionType() === this.#rawBlocksCompression
if (!options.ignoreBlockIndexes) {
await this.#vhd.readBlockAllocationTable()
@@ -277,7 +263,10 @@ export class RemoteVhdDisk extends RemoteDisk {
if (this.#vhd === undefined) {
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
}
await this.#vhd.writeBlockData(diskBlock.index, diskBlock.data)
await this.#vhd.writeEntireBlock({
id: diskBlock.index,
buffer: Buffer.concat([Buffer.alloc(this.#bitmapSize, 255), diskBlock.data]),
})
return this.getBlockSize()
}
@@ -285,22 +274,12 @@ export class RemoteVhdDisk extends RemoteDisk {
/**
* Reads a specific block from the VHD.
* @param {number} index
* @returns {Promise<DiskBlock | CompressedDiskBlock>} diskBlock
* @returns {Promise<DiskBlock>} diskBlock
*/
async readBlock(index) {
if (this.#vhd === undefined) {
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
}
if (this.#rawBlocks) {
/** @type {CompressedDiskBlock} */
const block = {
index,
// only a VHD directory has a compression, checked in init()
data: await /** @type {VhdDirectory} */ (this.#vhd).readCompressedBlock(index),
compression: /** @type {string} */ (this.#rawBlocksCompression),
}
return block
}
const { data } = await this.#vhd.readBlock(index)
return {
index,

View File

@@ -8,8 +8,8 @@
"dependencies": {
"@vates/async-each": "^1.0.3",
"@xen-orchestra/async-map": "^0.1.5",
"@xen-orchestra/backups": "^0.78.0",
"@xen-orchestra/fs": "^4.10.0",
"@xen-orchestra/backups": "^0.76.0",
"@xen-orchestra/fs": "^4.9.5",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",
"lodash": "^4.18.0",
@@ -28,7 +28,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "1.1.18",
"version": "1.1.16",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -2,7 +2,6 @@ import assert from 'node:assert'
import { formatFilenameDate } from './_filenameDate.mjs'
import { importIncrementalVm } from './_incrementalVm.mjs'
import { liveMountXapiLabels } from './liveMountXapiLabels.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { decorateClass } from '@vates/decorate-with'
import { Task } from '@vates/task'
@@ -43,8 +42,7 @@ export class ImportVmBackup {
* @param {object} [params.liveMount] - how to serve a disk from the backup repository instead of
* copying it, injected by the caller since a live mount outlives the restore and cannot be run
* from this package
* @param {({ cacheSrUuid, diskPath, hostId, xapiLabels }) => Promise<{ id: string, vdiUuid: string }>} params.liveMount.mountDisk
* `xapiLabels` are those of {@link liveMountXapiLabels}, `cacheSrUuid` makes the mount read/write
* @param {({ diskPath, hostId }) => Promise<{ id: string, vdiUuid: string }>} params.liveMount.mountDisk
* @param {(id: string) => Promise<void>} params.liveMount.unmountDisk
* @param {object} [params.settings]
*/
@@ -358,23 +356,11 @@ export class ImportVmBackup {
}
const diskPath = join(metadataDir, metadata.vhds[vdiRef])
const cacheSrUuid = this._vdiRestoreTargets.get(vdi.uuid).cacheSr
const readWrite = cacheSrUuid !== undefined
const xapiLabels = liveMountXapiLabels({
readWrite,
timestamp: metadata.timestamp,
vdiNameLabel: vdi.name_label,
vmNameLabel: metadata.vm.name_label,
})
const mount = await liveMount.mountDisk({ cacheSrUuid, diskPath, hostId, xapiLabels })
const mount = await liveMount.mountDisk({ diskPath, hostId })
this.#liveMounts.push({ ...mount, hostId })
info('disk live mounted', { cacheSrUuid, diskPath, hostId, mountId: mount.id, vdiUuid: vdi.uuid })
info('disk live mounted', { diskPath, hostId, mountId: mount.id, vdiUuid: vdi.uuid })
const record = {
...vdi,
liveMountedVdiRef: await xapi.call('VDI.get_by_uuid', mount.vdiUuid),
liveMountedReadOnly: !readWrite,
}
const record = { ...vdi, liveMountedVdiRef: await xapi.call('VDI.get_by_uuid', mount.vdiUuid) }
// it is attached as is: there is nothing to clone it from, nothing to transfer into it, and
// no SR to create it on — `SR` still holds the ref it had on the backed up pool
delete record.baseVdi

View File

@@ -141,20 +141,7 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
const backup = await importer._decorateIncrementalVmMetadata()
assert.deepEqual(liveMount.mounted, [
{
// not cached: read only
cacheSrUuid: undefined,
diskPath: `/xo-vm-backups/${VM_UUID}/${vhdPath(DATA)}`,
hostId: HOST,
// named after the backed up VM and disk, not after the file on the backup repository
xapiLabels: {
srNameLabel: '[XO live mount] a vm (20250801T080832Z)',
vdiNameLabel: 'data',
vdiNameDescription: 'Read-only live mount of a vm (20250801T080832Z), removed on unmount.',
},
},
])
assert.deepEqual(liveMount.mounted, [{ diskPath: `/xo-vm-backups/${VM_UUID}/${vhdPath(DATA)}`, hostId: HOST }])
// no data is transferred for it
assert.deepEqual(Object.keys(backup.disks), [SYSTEM.ref])
})
@@ -166,7 +153,6 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
const vdi = backup.vdis[DATA.ref]
assert.equal(vdi.liveMountedVdiRef, 'VDI_REF:mounted-vdi-1')
assert.equal(vdi.liveMountedReadOnly, true)
assert.equal(vdi.SR, undefined)
assert.equal(vdi.uuid, DATA.uuid)
// the other disk is restored as usual
@@ -174,19 +160,6 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
assert.equal(backup.vdis[SYSTEM.ref].liveMountedVdiRef, undefined)
})
it('mounts a cached disk read/write, and says so in its names', async () => {
const { importer, liveMount } = makeImporter({
mapVdisSrs: { [DATA.uuid]: { type: 'live-mount', host: HOST, cacheSr: OTHER_SR } },
})
const backup = await importer._decorateIncrementalVmMetadata()
const [mounted] = liveMount.mounted
assert.equal(mounted.cacheSrUuid, OTHER_SR)
assert.equal(mounted.xapiLabels.srNameLabel, '[XO live restore] a vm (20250801T080832Z)')
assert.equal(backup.vdis[DATA.ref].liveMountedReadOnly, false)
})
it('rejects live mounts spread over several hosts', async () => {
const { importer } = makeImporter({
mapVdisSrs: {

View File

@@ -40,10 +40,10 @@ afterEach(async () => {
const uniqueId = () => uuid.v1()
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
async function getAdapter({ useVhdDirectory = false, remoteCompressionType } = {}) {
async function getAdapter({ useVhdDirectory = false, vhdDirectoryCompression, remoteCompressionType } = {}) {
handler = getHandler({ url: `file://${tempDir}`, useVhdDirectory, compressionType: remoteCompressionType })
await handler.sync()
return new RemoteAdapter(handler)
return new RemoteAdapter(handler, { vhdDirectoryCompression })
}
const listVmDir = () => handler.list(rootPath, { ignoreMissing: true })
@@ -145,8 +145,7 @@ describe('RemoteAdapter#isMergeableParent', { concurrency: 1 }, () => {
test(`diskIsDirectory=${diskIsDirectory} useVhdDirectory=${useVhdDirectory} compressionMatches=${compressionMatches} uuidMatches=${uuidMatches} -> ${expected}`, async () => {
const targetCompression = 'brotli'
// no compressionType on the remote: brotli
const adapter = await getAdapter({ useVhdDirectory })
const adapter = await getAdapter({ useVhdDirectory, vhdDirectoryCompression: targetCompression })
const targetUuid = uniqueIdBuffer()
await generateVhd(`${basePath}/disk.vhd`, {
@@ -165,32 +164,37 @@ describe('RemoteAdapter#isMergeableParent', { concurrency: 1 }, () => {
})
describe('RemoteAdapter#isMergeableParent compression resolution', { concurrency: 1 }, () => {
test('the compressionType of the remote is the one compared against', async () => {
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
test('the per-remote compressionType override wins over the adapter default', async () => {
const adapter = await getAdapter({
useVhdDirectory: true,
vhdDirectoryCompression: 'gzip', // config.toml-style default
remoteCompressionType: 'brotli', // per-remote override, should be the one actually compared against
})
const targetUuid = uniqueIdBuffer()
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'gzip', uuid: targetUuid })
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), true)
})
test('a disk compressed with another codec than the remote is rejected', async () => {
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
test('a disk compressed with the adapter default (not the override) is rejected', async () => {
const adapter = await getAdapter({
useVhdDirectory: true,
vhdDirectoryCompression: 'gzip',
remoteCompressionType: 'brotli',
})
const targetUuid = uniqueIdBuffer()
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
// disk uses the config default, not the remote override that's actually in effect
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'gzip', uuid: targetUuid })
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), false)
})
test('a remote without compressionType uses brotli', async () => {
const adapter = await getAdapter({ useVhdDirectory: true })
const targetUuid = uniqueIdBuffer()
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), true)
})
test("a remote-level 'none' means no compression", async () => {
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'none' })
test("a remote-level 'none' override means no compression, ignoring the adapter default", async () => {
const adapter = await getAdapter({
useVhdDirectory: true,
vhdDirectoryCompression: 'brotli',
remoteCompressionType: 'none',
})
const targetUuid = uniqueIdBuffer()
// no compression option => VhdDirectory's own getCompressionType() is undefined, matching 'none'
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', uuid: targetUuid })
@@ -319,105 +323,3 @@ describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, ()
})
}
})
// IncrementalRemote runner when source and targets use the same compression: the blocks are copied as stored
describe('RemoteAdapter raw blocks copy', { concurrency: 1 }, () => {
// copy a disk the way the IncrementalRemote runner does, and check the target content
async function copyAndCheck(adapter, sourcePath, { rawBlocksCompression, expectedCompression }) {
const targetPath = `${basePath}/${adapter.getVhdFileName('target')}`
const source = await adapter._createVhdDisk(handler, sourcePath, { useChain: false, rawBlocksCompression })
await adapter.writeVhd(targetPath, source, {
validator: tmpPath => checkDisk(handler, tmpPath),
uuid: uniqueIdBuffer(),
})
await Disposable.use(openVhd(handler, targetPath), async vhd => {
assert.equal(vhd.compressionType, expectedCompression)
await vhd.readBlockAllocationTable()
assert.ok(vhd.containsBlock(0))
assert.ok(!vhd.containsBlock(1))
const { data } = await vhd.readBlock(2)
assert.ok(
data.every(byte => byte === 0x42),
'block 2 should contain the source disk data'
)
})
}
for (const compression of ['brotli', 'gzip', 'none']) {
test(`${compression}: a disk read with the same compression is copied as stored, readable and identical`, async () => {
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: compression })
const sourcePath = `${basePath}/${adapter.getVhdFileName('source')}`
await adapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
const source = await adapter._createVhdDisk(handler, sourcePath, {
useChain: false,
rawBlocksCompression: compression,
})
const block = await source.readBlock(2)
await source.close()
assert.equal(block.compression, compression)
if (compression !== 'none') {
assert.ok(block.data.length < 2 * 1024 * 1024, 'the block should be read compressed')
}
await copyAndCheck(adapter, sourcePath, { rawBlocksCompression: compression, expectedCompression: compression })
})
}
test('a disk written before a compression change is read normally and copied', async () => {
const gzipAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
const sourcePath = `${basePath}/${gzipAdapter.getVhdFileName('source')}`
await gzipAdapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
await handler.forget()
const brotliAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
const source = await brotliAdapter._createVhdDisk(handler, sourcePath, {
useChain: false,
rawBlocksCompression: 'brotli',
})
const block = await source.readBlock(2)
await source.close()
assert.equal(block.compression, undefined)
assert.equal(block.data.length, 2 * 1024 * 1024)
await copyAndCheck(brotliAdapter, sourcePath, { rawBlocksCompression: 'brotli', expectedCompression: 'brotli' })
})
test('a VHD file written before the switch to VHD directories is read normally and copied', async () => {
const fileAdapter = await getAdapter({ useVhdDirectory: false })
const sourcePath = `${basePath}/${fileAdapter.getVhdFileName('source')}`
await fileAdapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
await handler.forget()
const directoryAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
await copyAndCheck(directoryAdapter, sourcePath, { rawBlocksCompression: 'brotli', expectedCompression: 'brotli' })
})
// blocks as read from a VHD directory using `compression`, without their real content: only the checks are tested
class MockCompressedDisk extends MockDisk {
#compression
constructor(compression, ...args) {
super(...args)
this.#compression = compression
}
async readBlock(index) {
return { ...(await super.readBlock(index)), compression: this.#compression }
}
}
test('a VHD directory target refuses blocks compressed differently', async () => {
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
await assert.rejects(
adapter.writeVhd(
`${basePath}/${adapter.getVhdFileName('target')}`,
new MockCompressedDisk('gzip', 1, [0], 0x42),
{
uuid: uniqueIdBuffer(),
}
),
/can't write a gzip compressed block 0/
)
})
})

View File

@@ -3,7 +3,6 @@ import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
import { createLogger } from '@xen-orchestra/log'
import { stringify } from 'uuid'
import { decorateMethodsWith } from '@vates/decorate-with'
import assert from 'node:assert'
import { basename, dirname, join, resolve } from 'node:path'
import { normalize } from '@xen-orchestra/fs/path'
import { synchronized } from 'decorator-synchronized'
@@ -27,7 +26,6 @@ import { formatFilenameDate } from './_filenameDate.mjs'
import { isMetadataFile } from './_backupType.mjs'
import {
isKnownJournalEvent,
purgeBackupJournal,
readBackupJournal,
writeBackupJournalEntries,
writeBackupJournalEntry,
@@ -66,10 +64,6 @@ export class RemoteAdapter {
this._handler = handler
this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
this._useGetDiskLegacy = useGetDiskLegacy
// Cache of the partition list per diskId, populated by listPartitions and evicted by
// getDisk when the disk is unmounted
this._partitionsCache = new Map()
}
get handler() {
@@ -323,14 +317,6 @@ export class RemoteAdapter {
return this.handler.getConfig('useVhdDirectory')
}
/**
* @returns {string | undefined} the compression of the VHD directories written on this remote ('none' if not
* compressed), undefined if this remote writes plain VHD files
*/
getVhdDirectoryCompression() {
return this.useVhdDirectory() ? (this.handler.getConfig('compressionType') ?? 'brotli') : undefined // compatibility layer
}
#useAlias() {
return this.useVhdDirectory()
}
@@ -560,14 +546,6 @@ export class RemoteAdapter {
return readBackupJournal(this._handler, cursor, opts)
}
/**
* @param {number} before timestamp in ms, the journal days before its own are removed
* @returns {Promise<void>}
*/
async purgeBackupJournal(before) {
return purgeBackupJournal(this._handler, before)
}
// Same as `readBackupJournal()`, with the current metadata of the added and changed backups
// attached, so that a listing can be brought up to date from the result alone, in a single
// round-trip for a caller which is not on this host.
@@ -698,7 +676,7 @@ export class RemoteAdapter {
path,
concurrency: writeBlockConcurrency,
validator,
compression: this.getVhdDirectoryCompression(),
compression: handler.getConfig('compressionType') ?? 'brotli', // compatibility layer
uuid,
parentUuid,
parentPath,
@@ -724,8 +702,7 @@ export class RemoteAdapter {
) {
const container = watchStreamSize(input)
await this._handler.outputStream(path, input, {
// a known checksum is the one of unencrypted data: an encrypted remote stores no checksum
checksum: typeof checksum === 'string' && this._handler.isEncrypted ? false : checksum,
checksum,
dirMode: this._dirMode,
maxStreamLength,
streamLength,
@@ -738,24 +715,19 @@ export class RemoteAdapter {
}
// open the hierarchy of ancestors until we find a full one
async _createVhdDisk(handler, path, { useChain, rawBlocksCompression }) {
async _createVhdDisk(handler, path, { useChain }) {
let disk
if (useChain) {
assert.strictEqual(rawBlocksCompression, undefined, `can't read raw blocks of a disk chain`)
disk = await openDiskChain({ handler, path })
} else {
disk = new RemoteVhdDisk({ handler, path, rawBlocksCompression })
disk = new RemoteVhdDisk({ handler, path })
await disk.init()
}
disk = new ReadAhead(disk)
return disk
}
/**
* `rawBlocksCompression`: the blocks of the VHD directories using this compression are read as stored (see
* RemoteVhdDisk), to be written as is in a remote using the same compression
*/
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true, rawBlocksCompression } = {}) {
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
const handler = this._handler
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
const dir = dirname(metadata._filename)
@@ -763,7 +735,7 @@ export class RemoteAdapter {
const disks = {}
await asyncMapSettled(Object.keys(vdis), async ref => {
delete vdis[ref].baseVdi
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain, rawBlocksCompression })
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
})
return {
@@ -781,28 +753,6 @@ export class RemoteAdapter {
return this._handler.createReadStream(resolve('/', dirname(metadata._filename), metadata.xva))
}
/**
* @returns {Promise<string | undefined>} the checksum of the XVA as stored, undefined if it is unknown (no checksum
* file, or encrypted remote: the stored data are not the data read)
*/
async readFullVmBackupChecksum(metadata) {
if (this._handler.isEncrypted) {
return
}
try {
const checksum = String(
await this._handler.readFile(resolve('/', dirname(metadata._filename), metadata.xva) + '.checksum')
).trim()
// $<algorithm id>$<salt>$<hash>
return checksum.startsWith('$') ? checksum : undefined
} catch (error) {
if (error.code === 'ENOENT') {
return
}
throw error
}
}
async readVmBackupMetadata(path) {
let json
let isImmutable = false

View File

@@ -393,47 +393,6 @@ describe('readBackupJournal()', { concurrency: 1 }, () => {
})
})
describe('purgeBackupJournal()', { concurrency: 1 }, () => {
const DAY = 24 * 60 * 60 * 1e3
const HOUR = 60 * 60 * 1e3
// midnight of the given number of days ago, so that the tests do not depend on the time they run at
const daysAgo = n => (Math.floor(Date.now() / DAY) - n) * DAY
const writeEntryAt = timestamp =>
writeJournalEntryAt(timestamp, { event: 'add', vmUuid, filename: `/${rootPath}/${timestamp}.json` })
test('removes the days before the one of `before`, and keeps that day whole', async () => {
for (const n of [3, 2]) {
await writeEntryAt(daysAgo(n) + HOUR)
}
// after the entry of its own day: that entry is kept all the same
await adapter.purgeBackupJournal(daysAgo(2) + 2 * HOUR)
assert.deepEqual(await listJournalDays(), [formatJournalDay(daysAgo(2))])
})
test('skips a day which cannot be removed, and removes the next ones', async t => {
for (const n of [3, 2]) {
await writeEntryAt(daysAgo(n))
}
const lockedDay = formatJournalDay(daysAgo(3))
const rmtree = handler.rmtree.bind(handler)
t.mock.method(handler, 'rmtree', async path => {
if (path === `/${BACKUP_JOURNAL_DIR}/${lockedDay}`) {
throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' })
}
return rmtree(path)
})
await adapter.purgeBackupJournal(Date.now())
assert.deepEqual(await listJournalDays(), [lockedDay])
})
})
describe('readBackupJournalEvents()', { concurrency: 1 }, () => {
// records the paths whose metadata is read from now on, to check a replay does not re-read the
// same backup once per event

View File

@@ -309,47 +309,3 @@ export async function readBackupJournal(handler, cursor, { mustExist = false } =
return entries
}
/**
* Removes the journal entries of the days before the one of `before`.
*
* Whole days are removed, entries are never removed one by one: the day of `before` is kept entirely,
* whatever the time of its entries.
*
* A day which cannot be removed is skipped, e.g. its files are immutable (`EPERM` on a local
* repository, `AccessDenied` from the policy of a bucket): it must not keep the later days from being
* removed, and the next purge tries it again. Only a failure to list the journal rejects.
*
* A purge warns once, with its first failure and their count: where deletes are refused, every day
* before `before` fails again on each purge, one more each day.
*
* @param {RemoteHandler} handler
* @param {number} before timestamp in ms
* @returns {Promise<void>}
*/
export async function purgeBackupJournal(handler, before) {
const minDay = formatJournalDay(before)
const days = await handler.list(`/${BACKUP_JOURNAL_DIR}`, {
filter: name => isDayName(name) && name < minDay,
ignoreMissing: true,
})
let firstFailure
let nFailures = 0
// one day at a time: this runs in the background, it must not compete with the backups for the
// remote
for (const day of days) {
const path = `/${BACKUP_JOURNAL_DIR}/${day}`
try {
await handler.rmtree(path)
debug('journal day purged', { path })
} catch (error) {
if (firstFailure === undefined) {
firstFailure = { error, path }
}
++nFailures
}
}
if (firstFailure !== undefined) {
warn('failed to purge some journal days', { ...firstFailure, nFailures })
}
}

View File

@@ -75,6 +75,7 @@ class BackupWorker {
yield new RemoteAdapter(handler, {
debounceResource: this.debounceResource,
dirMode: this.#config.dirMode,
vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
})
} finally {
await handler.forget()

View File

@@ -1,111 +0,0 @@
// @ts-check
import { createLogger } from '@xen-orchestra/log'
import moment from 'moment-timezone'
import { parse } from 'xo-remote-parser'
const { warn } = createLogger('xo:backups:buildContext')
/**
* @typedef {import('@vates/types/xen-api').XenApiVm} XenApiVm
* @typedef {import('@vates/types/xo').XoSr} XoSr
* @typedef {import('@vates/types/xo').XoBackupRepository} XoBackupRepository
*
* @typedef {object} RunContext
* @property {number} dayOfWeek
* @property {number} dayOfMonth
* @property {number} hour
* @property {number} month
* @property {number} year
*
* @typedef {object} VmContext
* @property {Record<string, unknown>} vm
* @property {RunContext} run
* @property {number} [chainLength]
*
* @typedef {object} SrContext
* @property {{name_label: string, type: string, description: string, tags: string[]}} sr
* @property {RunContext} run
*
* @typedef {object} BackupRepositoryContext
* @property {{name: string, type: string | undefined, tags: string[]}} backupRepository
* @property {RunContext} run
*/
/**
* @param {Date} date
* @param {string | undefined} timezone
* @returns {RunContext}
*/
export function buildRunContext(date, timezone) {
const momentDate = timezone ? moment.tz(date, timezone) : moment(date)
return {
dayOfWeek: momentDate.day(),
dayOfMonth: momentDate.date(),
hour: momentDate.hour(),
month: momentDate.month() + 1,
year: momentDate.year(),
}
}
/**
* @param {XenApiVm} vm
* @param {RunContext} run
* @param {number} [chainLength]
* @returns {VmContext}
*/
export function buildVmContext(vm, run, chainLength) {
/** @type {Record<string, unknown>} */
const vmObject = {}
for (const [key, value] of Object.entries(vm)) {
if (!key.startsWith('$') && typeof value !== 'function') {
vmObject[key] = value
}
}
if (!Array.isArray(vmObject.tags)) {
vmObject.tags = []
}
return { vm: vmObject, run, chainLength }
}
/**
* @param {XoSr} sr
* @param {RunContext} run
* @returns {SrContext}
*/
export function buildSrContext(sr, run) {
return {
sr: {
name_label: sr.name_label,
type: sr.SR_type,
description: sr.name_description,
tags: Array.isArray(sr.tags) ? sr.tags : [],
},
run,
}
}
/**
* @param {XoBackupRepository} backupRepository
* @param {RunContext} run
* @returns {BackupRepositoryContext}
*/
export function buildBackupRepositoryContext(backupRepository, run) {
let type
try {
type = parse(backupRepository.url).type
} catch (error) {
warn('cannot parse backup repository URL', { backupRepositoryId: backupRepository.id, error })
}
return {
backupRepository: {
name: backupRepository.name,
type,
tags: [], // Placeholder - Backup repositories don't have tags yet
},
run,
}
}

View File

@@ -1,147 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { buildRunContext, buildVmContext, buildSrContext, buildBackupRepositoryContext } from './_buildContext.mjs'
import { BACKUP_EXPRESSION_CONTEXT } from '@vates/types/backup-expression-context'
describe('buildContext', () => {
const date = new Date('2025-01-05T03:00:00Z')
const runContext = buildRunContext(date, 'America/New_York')
const expectedRunContext = {
dayOfWeek: 6,
dayOfMonth: 4,
hour: 22,
month: 1,
year: 2025,
}
it('buildRunContext', () => {
assert.deepEqual(runContext, expectedRunContext)
})
it('buildVmContext', () => {
const vm1 = { name: 'name', $value: '$value', func: () => {} }
const vmContext1 = buildVmContext(vm1, runContext, 3)
assert.deepEqual(vmContext1, {
vm: {
name: 'name',
tags: [],
},
run: expectedRunContext,
chainLength: 3,
})
const vm2 = { name: 'name', $value: '$value', func: () => {}, tags: ['tag'] }
const vmContext2 = buildVmContext(vm2, runContext, undefined)
assert.deepEqual(vmContext2, {
vm: {
name: 'name',
tags: ['tag'],
},
run: expectedRunContext,
chainLength: undefined,
})
const vm3 = { name: 'name', $value: '$value', func: () => {}, tags: 'tag' }
const vmContext3 = buildVmContext(vm3, runContext, undefined)
assert.deepEqual(vmContext3, {
vm: {
name: 'name',
tags: [],
},
run: expectedRunContext,
chainLength: undefined,
})
})
it('buildVmContext exposes every field advertised by the schema', () => {
// vm is an open context (buildVmContext copies through all real VM fields),
// so we assert the schema's curated fields are a subset of the output, not equality.
const vm = { name_label: 'name', power_state: 'Running', tags: ['tag'], $ref: 'ref', func: () => {} }
const { vm: vmContext } = buildVmContext(vm, runContext)
for (const field of Object.keys(BACKUP_EXPRESSION_CONTEXT.vm)) {
assert.ok(field in vmContext, `vm context is missing schema field: ${field}`)
}
})
it('buildSrContext', () => {
const sr1 = {
name_label: 'name_label',
SR_type: 'type',
name_description: 'description',
ignored: 'ignored',
}
const srContext1 = buildSrContext(sr1, runContext)
assert.deepEqual(srContext1, {
sr: {
name_label: 'name_label',
type: 'type',
description: 'description',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(Object.keys(srContext1.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
const sr2 = {
name_label: 'name_label',
SR_type: 'type',
name_description: 'description',
tags: ['tag'],
ignored: 'ignored',
}
const srContext2 = buildSrContext(sr2, runContext)
assert.deepEqual(srContext2, {
sr: {
name_label: 'name_label',
type: 'type',
description: 'description',
tags: ['tag'],
},
run: expectedRunContext,
})
assert.deepEqual(Object.keys(srContext2.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
})
it('buildBackupRepositoryContext', () => {
const backupRepository1 = {
name: 'name',
url: 'nfs://host/path',
ignored: 'ignored',
}
const backupRepositoryContext1 = buildBackupRepositoryContext(backupRepository1, runContext)
assert.deepEqual(backupRepositoryContext1, {
backupRepository: {
name: 'name',
type: 'nfs',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(
Object.keys(backupRepositoryContext1.backupRepository).sort(),
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
)
const backupRepository2 = {
name: 'name',
url: 'nfs://host/path',
tags: [],
ignored: 'ignored',
}
const backupRepositoryContext2 = buildBackupRepositoryContext(backupRepository2, runContext)
assert.deepEqual(backupRepositoryContext2, {
backupRepository: {
name: 'name',
type: 'nfs',
tags: [],
},
run: expectedRunContext,
})
assert.deepEqual(
Object.keys(backupRepositoryContext2.backupRepository).sort(),
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
)
})
})

View File

@@ -1,24 +0,0 @@
// @ts-check
import * as CM from 'complex-matcher'
/**
* Parse and compile a complex-matcher expression into a predicate function.
* Throws synchronously if the expression is syntactically invalid.
*
* @param {string} expression
* @returns {(context: object) => boolean}
*/
export function compileExpression(expression) {
return CM.parse(expression).createPredicate()
}
/**
* Returns true if value is a non-empty string — i.e. a complex-matcher expression.
*
* @param {unknown} value
* @returns {value is string}
*/
export function isExpression(value) {
return typeof value === 'string' && value.length > 0 && !Number.isFinite(Number(value))
}

View File

@@ -1,95 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { compileExpression, isExpression } from './_expressionPredicate.mjs'
describe('compileExpression', () => {
it('run:dayOfWeek:0', () => {
const predicate = compileExpression('run:dayOfWeek:0')
assert(predicate({ run: { dayOfWeek: 0 } }))
assert(!predicate({ run: { dayOfWeek: 1 } }))
})
it('chainLength:>5', () => {
const predicate = compileExpression('chainLength:>5')
assert(predicate({ chainLength: 6 }))
assert(!predicate({ chainLength: 5 }))
})
it('vm:tags:prod', () => {
const predicate = compileExpression('vm:tags:prod')
assert(predicate({ vm: { tags: ['prod'] } }))
assert(predicate({ vm: { tags: ['prod', 'test'] } }))
assert(!predicate({ vm: { tags: ['test'] } }))
assert(!predicate({ vm: { tags: [] } }))
})
it('run:dayOfWeek:0 chainLength:>3', () => {
const predicate = compileExpression('run:dayOfWeek:0 chainLength:>3')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ run: { dayOfWeek: 0 }, chainLength: 2 }))
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
})
it('|(run:dayOfWeek:0 chainLength:>6)', () => {
const predicate = compileExpression('|(run:dayOfWeek:0 chainLength:>6)')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 7 }))
assert(predicate({ run: { dayOfWeek: 1 }, chainLength: 7 }))
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
})
it('missing field', () => {
const predicate = compileExpression('run:dayOfWeek:0')
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
assert(!predicate({ chainLength: 6 }))
assert(!predicate({ run: {}, chainLength: 6 }))
})
it('null-checking', () => {
const predicate = compileExpression('!vm')
assert(predicate({}))
assert(!predicate({ vm: [{ vm: 'vm' }] }))
})
it('throws on invalid expression', () => {
assert.throws(() => compileExpression('vm:('))
})
})
describe('isExpression', () => {
it('non-empty string', () => {
assert(isExpression('vm:tags:test'))
})
it('number', () => {
assert(!isExpression(0))
})
it('string[]', () => {
assert(!isExpression(['string']))
})
it('undefined', () => {
assert(!isExpression(undefined))
})
it('empty string', () => {
assert(!isExpression(''))
})
it('number', () => {
assert(!isExpression('3'))
})
it('number with spaces', () => {
assert(!isExpression(' 5 '))
})
})

View File

@@ -1,7 +1,7 @@
import { strict as assert } from 'node:assert'
import test from 'node:test'
import { execFile } from 'node:child_process'
import { copyFile, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { Stream } from 'node:stream'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -9,7 +9,6 @@ import { promisify } from 'node:util'
import { randomBytes } from 'node:crypto'
import Disposable from 'promise-toolbox/Disposable'
import * as tar from 'tar'
import { RemoteAdapter } from './RemoteAdapter.mjs'
@@ -72,83 +71,6 @@ test('fetchPartitionFiles resolves with a node:stream for every format', async t
}
})
async function readAll(stream) {
const chunks = []
for await (const chunk of stream) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
async function listTgzEntries(archive, tmp) {
const file = join(tmp, 'archive.tgz')
await writeFile(file, archive)
const entries = []
await tar.t({ file, onReadEntry: entry => entries.push(entry.path.replace(/\/$/, '')) })
return entries
}
// The partition is mounted on the XOA/proxy: a guest symlink is resolved against the host root,
// so following one would expose host files. Paths through a symlink are refused, and symlinks
// met while walking a directory are skipped.
test('file restore never follows symbolic links', async t => {
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-symlink-'))
t.after(() => rm(tmp, { recursive: true, force: true }))
// `host` stands for the XOA filesystem, `root` for the mounted guest partition
const host = join(tmp, 'host')
const root = join(tmp, 'root')
await mkdir(host)
await writeFile(join(host, 'secret.txt'), 'host secret')
await mkdir(join(root, 'dir'), { recursive: true })
await writeFile(join(root, 'dir', 'file.txt'), 'guest file')
await symlink(host, join(root, 'absolute')) // like `ln -s / foo` in the guest
await symlink('../../host', join(root, 'dir', 'relative'))
await symlink('file.txt', join(root, 'dir', 'in-tree'))
const adapter = new RemoteAdapter({})
adapter.getPartition = Disposable.factory(async function* () {
yield root
})
const refusedPaths = ['/absolute', '/absolute/secret.txt', '/dir/relative', '/dir/relative/host/secret.txt']
await t.test('listing refuses a path through a symlink', async () => {
for (const path of refusedPaths) {
await assert.rejects(adapter.listPartitionFiles('disk', 'partition', path), { code: 'ELINK_NOT_SUPPORTED' }, path)
}
})
await t.test('listing skips symlinks', async () => {
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/'), { 'dir/': {} })
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/dir'), { 'file.txt': {} })
})
for (const format of ['tgz', 'zip']) {
await t.test(`${format}: download refuses a path through a symlink`, async () => {
for (const path of refusedPaths) {
await assert.rejects(
adapter.fetchPartitionFiles('disk', 'partition', ['/dir/file.txt', path], format),
{ code: 'ELINK_NOT_SUPPORTED' },
path
)
}
})
}
await t.test('tgz: symlinks inside a selected directory are skipped', async () => {
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'tgz'))
assert.deepEqual((await listTgzEntries(archive, tmp)).sort(), ['dir', 'dir/file.txt'])
})
await t.test('zip: symlinks inside a selected directory are skipped', async () => {
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'zip'))
assert.ok(archive.includes('dir/file.txt'))
assert.equal(archive.includes('dir/relative'), false)
assert.equal(archive.includes('dir/in-tree'), false)
})
})
test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => {
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-'))
const adapter = new RemoteAdapter({})

View File

@@ -66,29 +66,7 @@ export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
}))
const makeRelative = path => resolve('/', path).slice(1)
// The partition is mounted on the XOA/proxy, so the kernel resolves a guest symlink against the
// host root: following one (e.g. `ln -s / foo` in the VM, then `/foo/etc`) would read host files.
// Symlinks are therefore not supported by file restore: a path going through one is refused,
// checked component by component with lstat. The mount is read-only, so no check/use race.
export async function resolveWithoutSymlinks(rootPath, path) {
const relativePath = makeRelative(path)
let currentPath = rootPath
for (const name of relativePath.split('/')) {
if (name === '') {
continue
}
currentPath = join(currentPath, name)
const stats = await lstat(currentPath)
if (stats.isSymbolicLink()) {
const error = new Error(`symbolic links are not supported by file restore: ${path}`)
error.code = 'ELINK_NOT_SUPPORTED'
error.path = path
throw error
}
}
return relativePath
}
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
for (const relativePath of relativePaths) {
@@ -312,12 +290,6 @@ export const fileRestoreMethods = {
const self = this
Disposable.use(async function* () {
const path = yield self.getPartition(diskId, partitionId)
// validated before resolving the promise, so a refused path rejects instead of
// producing a truncated archive
const relativePaths = []
for (const requestedPath of paths) {
relativePaths.push(await resolveWithoutSymlinks(path, requestedPath))
}
let outputStream
if (format === 'tgz') {
@@ -326,13 +298,7 @@ export const fileRestoreMethods = {
// simultaneous reads. Those saturate the libuv threadpool and starve
// the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
// threadpool deadlock). Serializing keeps a worker free for them.
//
// symlinks found while walking the selected directories are skipped, like in the zip
// format and the file listing
const pack = tar.c(
{ cwd: path, gzip: true, jobs: 1, filter: (_, stats) => !stats.isSymbolicLink() },
relativePaths
)
const pack = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
// node-tar >= 7 returns a Minipass stream: it has `pipe()` but is not a `node:stream`
// instance. xo-proxy serves this stream through Koa, which pipes only bodies passing
@@ -350,7 +316,7 @@ export const fileRestoreMethods = {
// appear as a freeze with no response sent.
outputStream = zip.outputStream
resolve(zip.outputStream)
await addZipEntries(zip, path, '', relativePaths)
await addZipEntries(zip, path, '', paths.map(makeRelative))
zip.end()
} else {
throw new Error('unsupported format ' + format)
@@ -365,23 +331,17 @@ export const fileRestoreMethods = {
},
async *getDisk(diskId) {
try {
if (this._useGetDiskLegacy) {
yield* getDiskLegacy(this._handler, diskId)
return
}
const handler = this._handler
// this is a disposable
const mountDir = yield getTmpDir()
// this is also a disposable
yield mount(handler, diskId, mountDir)
// this will yield disk path to caller
yield `${mountDir}/vhd0`
} finally {
// Disposable.factory disposes by calling gen.return(), which runs this finally before
// the mount/tmpdir disposers — i.e. exactly as the disk is unmounted.
this._partitionsCache.delete(diskId)
if (this._useGetDiskLegacy) {
yield* getDiskLegacy(this._handler, diskId)
return
}
const handler = this._handler
// this is a disposable
const mountDir = yield getTmpDir()
// this is also a disposable
yield mount(handler, diskId, mountDir)
// this will yield disk path to caller
yield `${mountDir}/vhd0`
},
// partitionId values:
@@ -411,7 +371,7 @@ export const fileRestoreMethods = {
listPartitionFiles(diskId, partitionId, path) {
return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
path = join(rootPath, await resolveWithoutSymlinks(rootPath, path))
path = resolveSubpath(rootPath, path)
const entriesMap = {}
await asyncEach(
await readdir(path),
@@ -437,13 +397,7 @@ export const fileRestoreMethods = {
},
listPartitions(diskId) {
const cache = this._partitionsCache
const cached = cache.get(diskId)
if (cached !== undefined) {
return cached
}
const pPartitions = Disposable.use(this.getDisk(diskId), async devicePath => {
return Disposable.use(this.getDisk(diskId), async devicePath => {
// partx may return empty on FUSE-backed files (vhd0); a loop device
// presents proper block-device semantics that partx reads reliably.
// losetup may itself fail if the FUSE mount isn't fully ready yet —
@@ -461,14 +415,11 @@ export const fileRestoreMethods = {
}
}
// direct listing failed, try with lvm listing
if (partitions.length === 0) {
debug('direct listing of the volume partition failed, try to read through LVM')
try {
// handle potential raw LVM physical volume
return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
} catch (error) {
warn('LVM listing of this volume failed ', error)
return []
}
}
@@ -504,16 +455,6 @@ export const fileRestoreMethods = {
})
return results
})
// The entry is evicted on disk unmount (see getDisk)
// a failed probe is not cached so the next call retries.
cache.set(diskId, pPartitions)
pPartitions.catch(() => {
if (cache.get(diskId) === pPartitions) {
cache.delete(diskId)
}
})
return pPartitions
},
}

View File

@@ -294,9 +294,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
await asyncMap(Object.values(vdiVbds), vbd =>
xapi.VBD_create({
...vbd,
// an uncached live mount serves its disk read only: attach it as such, instead of letting
// the guest discover it through I/O errors on its first write
mode: isLiveMounted && vdi.liveMountedReadOnly ? 'RO' : vbd.mode,
// a live mount serves its disk read only: attach it as such, instead of letting the
// guest discover it through I/O errors on its first write
mode: isLiveMounted ? 'RO' : vbd.mode,
VDI: newVdi.$ref,
VM: vmRef,
})
@@ -307,16 +307,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
})
// 3.5. Destroy old VDIs that are no longer attached to the VM.
// Some storage backends refuse to destroy a VDI that still has snapshot children.
// Log it and leave it for later cleanup.
await asyncMap([...oldVdiRefs], ref =>
xapi.call('VDI.destroy', ref).catch(error =>
Task.warning('failed to destroy orphaned VDI', {
vdi: xapi.getObject(ref, undefined)?.uuid ?? ref,
error,
})
)
)
// Uses ignoreErrors because some storage backends refuse to destroy a VDI that still has snapshot children;
// those VDIs will become truly orphaned once the old snapshot VMs are cleaned up by _deleteOldEntries.
await asyncMap([...oldVdiRefs], ref => ignoreErrors.call(xapi.call('VDI.destroy', ref)))
// 4. For updates, destroy existing VIFs before recreating them.
if (isUpdate) {

View File

@@ -60,7 +60,7 @@ const parsePartxLine = createParser({
export async function listPartitions(devicePath) {
const parts = await fromCallback(execFile, 'partx', [
'--bytes',
'--output=START,SIZE,NAME,UUID,TYPE',
'--output=NR,START,SIZE,NAME,UUID,TYPE',
'--pairs',
devicePath,
]).catch(error => {

View File

@@ -10,8 +10,6 @@ import { FullRemote } from './_vmRunners/FullRemote.mjs'
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
import { Throttle } from '@vates/generator-toolbox'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { compileExpression, isExpression } from '../_expressionPredicate.mjs'
import { buildRunContext, buildVmContext } from '../_buildContext.mjs'
const noop = Function.prototype
@@ -62,25 +60,21 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
return
}
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
const predicate = isExpression(settings.vmFilter) ? compileExpression(settings.vmFilter) : null
const vmsUuids = await sourceRemoteAdapter.listAllVms()
const queue = new Set(vmsUuids)
Task.info('vms', { vms: vmsUuids })
let nbVms = vmsUuids.length
const nbVms = vmsUuids.length
let nbVmsDone = 0
remoteAdapters = getAdaptersByRemote(remoteAdapters)
const allSettings = this._job.settings
const baseSettings = this._baseSettings
const queue = new Set(vmsUuids)
const taskByVmId = {}
const nTriesByVmId = {}
const handleVm = async vmUuid => {
const handleVm = vmUuid => {
if (nTriesByVmId[vmUuid] === undefined) {
nTriesByVmId[vmUuid] = 0
}
@@ -90,26 +84,12 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const vmSettings = { ...settings, ...allSettings[vmUuid] }
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
const vmBackups = await sourceRemoteAdapter.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
// avoiding to create tasks for empty directories
if (vmBackups.length === 0) {
nbVms--
return
}
if (predicate !== null && !predicate(buildVmContext(vmBackups.at(-1).vm, runContext))) {
nbVms--
return
}
const opts = {
baseSettings,
config,
job,
healthCheckSr,
remoteAdapters,
runContext,
schedule,
settings: vmSettings,
sourceRemoteAdapter,
@@ -126,39 +106,45 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
}
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
let taskError
return task
.runInside(async () =>
vmBackup.run().catch(error => {
taskError = error
})
)
.then(result => {
if (taskError === undefined) {
nbVmsDone++
return task.success(result)
return sourceRemoteAdapter
.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
.then(vmBackups => {
// avoiding to create tasks for empty directories
if (vmBackups.length > 0) {
if (taskByVmId[vmUuid] === undefined) {
taskByVmId[vmUuid] = new Task(taskStart)
}
const task = taskByVmId[vmUuid]
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
let taskError
return task
.runInside(async () =>
vmBackup.run().catch(error => {
taskError = error
})
)
.then(result => {
if (taskError === undefined) {
nbVmsDone++
return task.success(result)
}
if (isLastRun) {
nbVmsDone++
return task.failure(taskError)
}
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
// don't end the task
task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: taskError.message,
isRetry: true,
})
queue.add(vmUuid)
})
.catch(noop)
}
if (isLastRun) {
nbVmsDone++
return task.failure(taskError)
}
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
// don't end the task
task.warning(`Retry the VM mirror backup due to an error`, {
attempt: nTriesByVmId[vmUuid],
error: taskError.message,
isRetry: true,
})
queue.add(vmUuid)
})
.catch(noop)
}
const { concurrency } = settings
const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)

View File

@@ -12,7 +12,6 @@ import { Throttle } from '@vates/generator-toolbox'
import { asyncEach } from '@vates/async-each'
import createStreamThrottle from './_createStreamThrottle.mjs'
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
import { buildRunContext } from '../_buildContext.mjs'
const noop = Function.prototype
@@ -69,7 +68,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const getSnapshotNameLabel = this._getSnapshotNameLabel
const schedule = this._schedule
const settings = this._settings
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
const throttleGenerator = new Throttle(settings.maxExportRate)
const throttleStream = createStreamThrottle(settings.maxExportRate)
@@ -143,6 +141,8 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
await asyncEach(
[...vms].filter(vm => batchIds.has(vm.uuid)),
async vm => {
// The task deliverately includes both the transfer (including the wait) and and backup.
// This is done to have the snapshot reported under the VM.
const { task } = getVmTask(vm.uuid, vm.name_label)
const vmSettings = { ...settings, ...allSettings[vm.uuid] }
@@ -153,7 +153,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
healthCheckSr,
job,
remoteAdapters,
runContext,
schedule,
settings: vmSettings,
srs,
@@ -252,7 +251,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
throttleGenerator,
throttleStream,
vm,
runContext,
}
const vmBackup = this._getVmBackup(job.mode, opts)

View File

@@ -1,5 +1,3 @@
import { createLogger } from '@xen-orchestra/log'
import { AbstractRemote } from './_AbstractRemote.mjs'
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
@@ -7,8 +5,6 @@ import { watchStreamSize } from '../../_watchStreamSize.mjs'
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
import { Task } from '@vates/task'
const { warn } = createLogger('xo:backups:FullRemoteVmBackup')
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
_getRemoteWriters() {
return [FullRemoteWriter, AggregatedFullRemoteWriter]
@@ -25,23 +21,12 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
for (const metadata of transferList) {
const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
const sizeContainer = watchStreamSize(stream)
let checksum
try {
// the unencrypted destinations store the same data as the source: no need to compute their checksum again
checksum = await this._sourceRemoteAdapter.readFullVmBackupChecksum(metadata)
} catch (error) {
warn('error while copying reading source checksum, recomputing it', error)
}
// a single writer can consume the stream itself: no need to fork it
const isForked = this._writers.size > 1
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
await this._callWriters(
writer =>
writer.run({
checksum,
stream: isForked ? forkStreamUnpipe(stream) : stream,
stream: forkStreamUnpipe(stream),
// stream will be forked and transformed, it's not safe to attach additional properties to it
streamLength: stream.length,
maxStreamLength: stream.maxStreamLength, // for encrypted destination/source without length
@@ -53,7 +38,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
'writer.run()'
)
// for healthcheck
this._vm = metadata.vm
this._tags = metadata.vm.tags
nbTransferredVms++
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
}

View File

@@ -62,33 +62,14 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
})
// yeah , let's go
}
/**
* the block files can be copied as stored, never decompressed nor recompressed, if all the writers write VHD
* directories with the same compression (the source disks must also use it, checked per disk when reading)
* @returns {string | undefined} the common compression, undefined if a writer writes plain VHD files or if they differ
*/
#getRawBlocksCompression() {
const compressions = new Set([...this._writers].map(writer => writer.getVhdDirectoryCompression()))
// a single undefined value means plain VHD files on every writer: no raw blocks either
return compressions.size === 1 ? [...compressions][0] : undefined
}
async _run() {
const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
// the source remote writes with the same compression as all the writers: the blocks can be copied as stored
// this is based on the remote configurations, each source disk checks its own compression since it may predate a
// configuration change, and falls back to a normal read
const writersCompression = this.#getRawBlocksCompression()
const rawBlocksCompression =
writersCompression === this._sourceRemoteAdapter.getVhdDirectoryCompression() ? writersCompression : undefined
const nbTransferrableVms = transferList.length
let nbTransferredVms = 0
for (const metadata of transferList) {
assert.strictEqual(metadata.mode, 'delta')
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
useChain: false,
rawBlocksCompression,
})
// don't trust metadata too much
// recompute if it's a base backup
@@ -131,7 +112,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
// for healthcheck
this._vm = metadata.vm
this._tags = metadata.vm.tags
nbTransferredVms++
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
}

View File

@@ -18,19 +18,10 @@ import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
import { buildVmContext } from '../../_buildContext.mjs'
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
constructor(props) {
super(props)
const fullInterval = this._settings.fullInterval
this._fullIntervalPredicate = isExpression(fullInterval) ? compileExpression(fullInterval) : null
}
_getWriters() {
return [
IncrementalRemoteWriter,
@@ -238,17 +229,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
}
// we do tafter checkbasevdis because we want the writer to know the target VM
// especially on replication when we always update the same VM
// especially on replication when we alwayr update the same VM
const fullInterval = this._settings.fullInterval
const chainLength = (deltaChainLength ?? 0) + 1
if (isExpression(fullInterval)) {
const context = buildVmContext(this._vm, this._runContext, chainLength)
if (this._fullIntervalPredicate(context)) {
debug('forcing full backup: fullInterval expression matched', { fullInterval })
return
}
} else if (fullInterval !== 0 && fullInterval <= chainLength) {
debug('not using base VM because fullInterval reached', { fullInterval, chainLength })
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
debug('not using base VM because fullInterval reached', {
fullInterval,
deltaChainLength,
eq: fullInterval < deltaChainLength + 1,
dc1: deltaChainLength + 1,
})
return
}

View File

@@ -1,8 +1,6 @@
import { asyncMap } from '@xen-orchestra/async-map'
import { createLogger } from '@xen-orchestra/log'
import { Task } from '@vates/task'
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
import { buildVmContext } from '../../_buildContext.mjs'
const { debug, warn } = createLogger('xo:backups:AbstractVmRunner')
@@ -75,58 +73,24 @@ export const Abstract = class AbstractVmBackupRunner {
}
// check if current VM has tags
const tags = this._vm.tags
const tags = this._tags
// accept both 'xo:no-health-check' and 'xo:no-health-check=reason'
if (tags.some(t => t.startsWith('xo:no-health-check'))) {
const vmAlwaysIgnored = tags.some(t => t.startsWith('xo:no-health-check'))
const intersect = !vmAlwaysIgnored && settings.healthCheckVmsWithTags.some(t => tags.includes(t))
if (vmAlwaysIgnored || (settings.healthCheckVmsWithTags.length !== 0 && !intersect)) {
// create a task to have an info in the logs and reports
return Task.run(
{
properties: {
name: 'health check',
},
properties: { name: 'health check' },
},
() => {
Task.info(`This VM is excluded from health checks via tag`)
Task.info(`This VM doesn't match the health check's tags for this schedule`)
}
)
}
if (isExpression(settings.healthCheckVmsWithTags)) {
const context = buildVmContext(this._vm, this._runContext)
if (!this._healthCheckPredicate(context)) {
return Task.run(
{
properties: { name: 'health check' },
},
() => {
Task.info(`VM did not match healthCheckVmsWithTags expression`)
}
)
}
} else {
const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
// create a task to have an info in the logs and reports
return Task.run(
{
properties: { name: 'health check' },
},
() => {
Task.info(`This VM doesn't match the health check's tags for this schedule`)
}
)
}
}
await this._callWriters(writer => writer.healthCheck(), 'writer.healthCheck()')
}
_compileHealthCheckPredicate() {
const { healthCheckVmsWithTags } = this._settings
if (isExpression(healthCheckVmsWithTags)) {
this._healthCheckPredicate = compileExpression(healthCheckVmsWithTags)
} else {
this._healthCheckPredicate = undefined
}
}
}

View File

@@ -20,7 +20,6 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
job,
healthCheckSr,
remoteAdapters,
runContext,
schedule,
settings,
sourceRemoteAdapter,
@@ -41,14 +40,11 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
this._throttleGenerator = throttleGenerator
this._throttleStream = throttleStream
this._vmUuid = vmUuid
this._runContext = runContext
const allSettings = job.settings
const writers = new Set()
this._writers = writers
this._compileHealthCheckPredicate()
const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
if (settings.distributeBackups) {
writers.add(

View File

@@ -36,7 +36,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
job,
remoteAdapters,
remotes,
runContext,
schedule,
settings,
srs,
@@ -64,13 +63,12 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
this.timestamp = undefined
// VM currently backed up
const tags = vm.tags
const tags = (this._tags = vm.tags)
// VM (snapshot) that is really exported
this._exportedVm = undefined
this._synchronizedSnapshotTimestamp = synchronizedSnapshotTimestamp
this._vm = vm
this._runContext = runContext
this._getSnapshotNameLabel = getSnapshotNameLabel
this._isIncremental = job.mode === 'delta'
@@ -92,9 +90,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
settings.offlineSnapshot = true
}
this._settings = settings
this._compileHealthCheckPredicate()
// Create writers
{
const writers = new Set()

View File

@@ -13,9 +13,7 @@ export class TaskProgressHandler {
if (progress < 0 || progress > 1) {
return
}
// the end is always emitted: throttled, a last step under the threshold would leave the task at 99%
if (
progress !== 1 &&
this.#lastProgressDate !== undefined &&
this.#lastProgressValue !== undefined &&
Date.now() - this.#lastProgressDate < MAX_DURATION_BETWEEN_PROGRESS_EMIT &&

View File

@@ -1,15 +1,5 @@
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
export class AggregatedIncrementalRemoteWriter extends AbstractAggregatedRemoteWriter {
/**
* the backup can land on any of the adapters, the compression is only known if they all use the same one
* (based on the adapters since the child writers are only created in beforeBackup())
* @returns {string | undefined} see RemoteAdapter#getVhdDirectoryCompression
*/
getVhdDirectoryCompression() {
const compressions = new Set(Object.values(this.adapters).map(adapter => adapter.getVhdDirectoryCompression()))
return compressions.size === 1 ? [...compressions][0] : undefined
}
async deleteOldBackupsOnAdapter(adapter, backups) {
await adapter.deleteDeltaVmBackups(backups)
}

View File

@@ -6,14 +6,13 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
/**
*
* @param {Map<string,string>} baseUuidToSrcVdi
* @param {Map<string,string>} contentKeys
*/
async checkBaseVdis(baseUuidToSrcVdi, contentKeys) {
async checkBaseVdis(baseUuidToSrcVdi) {
debug('checkBaseVdis', { baseUuidToSrcVdi })
let selectedCopy = new Map()
for (const writer of this.writers) {
const copy = new Map(baseUuidToSrcVdi)
await writer.checkBaseVdis(copy, contentKeys)
await writer.checkBaseVdis(copy)
if (copy.size > 0) {
debug('checkBaseVdis found a mainwriter candidate ', writer._sr.name_label)
// there can be multiple candidates
@@ -35,17 +34,17 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
}
// take the lock on every adapters with this VM
// cleanup if deleteFirst is selected
async beforeBackup() {
await this.setupWriters()
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
this.setOldReplicaList()
if (this.props.settings.deleteFirst) {
await this.deleteOldReplicas()
}
}
async prepare(args) {
const { settings } = this.props
if (settings.deleteFirst) {
await this.deleteOldReplicas({ keepMostRecent: !args.isFull && settings.copyRetention === 1 })
}
if (this.mainWriter === undefined) {
debug('no mainwriter found, fallback to a new SR')
// use the SR with the most empty space for starting a new chain
@@ -63,7 +62,9 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
// remove the backups and remove the entries
async cleanup() {
debug('cleanup')
await this.deleteOldReplicas()
if (!this.props.settings.deleteFirst) {
await this.deleteOldReplicas()
}
await Promise.all(this.writers.map(writer => writer.cleanup()))
}

View File

@@ -25,11 +25,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
)
}
/**
* @param {object} params
* @param {string} [params.checksum] - checksum of the stream data if known (full mirror of an unencrypted backup)
*/
async _run({ checksum, maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
const settings = this._settings
const job = this._job
const scheduleId = this._schedule.id
@@ -71,7 +67,6 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
await Task.run({ properties: { name: 'transfer' } }, async () => {
await adapter.outputStream(dataFilename, stream, {
checksum,
maxStreamLength,
streamLength,
validator: tmpPath => adapter.isValidXva(tmpPath),

View File

@@ -25,14 +25,6 @@ const { warn } = createLogger('xo:backups:DeltaBackupWriter')
export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
#parentVdiPaths
#parentUuids
/**
* @returns {string | undefined} see RemoteAdapter#getVhdDirectoryCompression
*/
getVhdDirectoryCompression() {
return this._adapter.getVhdDirectoryCompression()
}
async checkBaseVdis(baseUuidToSrcVdi) {
this.#parentVdiPaths = {}
this.#parentUuids = {}

View File

@@ -65,11 +65,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
const snapshotCandidates = new Map()
// skip orphan snapshots (active VDI destroyed): they can't be validated against an active disk
const replicaSnapshots = sr.$VDIs.filter(
vdi => vdi?.managed && vdi?.is_a_snapshot && vdi.$snapshot_of !== undefined
)
for (const [baseUuid, srcVdiuid] of baseUuidToSrcVdi) {
let target
@@ -77,9 +72,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
if (contentKey !== undefined) {
debug('got one content key, look for a candidate')
target = replicaSnapshots.find(
target = sr.$VDIs.find(
vdi =>
vdi.other_config[CONTENT_KEY] === contentKey &&
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[CONTENT_KEY] === contentKey && // &&
// ensure we don't replicate on ourself or any of the vdi in the source chain
vdi.$snapshot_of.uuid !== srcVdiuid
)
@@ -88,11 +85,13 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
// fall back for older snapshots
if (target === undefined) {
debug('content key not here or not found , look by jobid ')
target = replicaSnapshots.find(
target = sr.$VDIs.find(
vdi =>
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[JOB_ID] === this._job.id &&
vdi.other_config[VM_UUID] === this._vmUuid &&
vdi.other_config[COPY_OF] === baseUuid
vdi?.other_config[COPY_OF] === baseUuid
)
}
if (target !== undefined) {
@@ -384,15 +383,14 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
retentionEntries.sort(compareReplicatedVmDatetime)
this._oldEntries = getOldEntries(settings.copyRetention - 1, retentionEntries)
if (settings.deleteFirst && !settings.skipDeleteOldEntries) {
if (settings.deleteFirst && settings.skipDeleteOldEntries) {
// we want to keep the baseVM when copying a delta
// even if we want to keep only one after
let mostRecentEntry
if (this._oldEntries.length > 0 && settings.copyRetention === 1 && !isFull) {
if (this._oldEntries.length > 1 && settings.copyRetention === 1 && !isFull) {
mostRecentEntry = this._oldEntries.pop()
}
await this._deleteOldEntries()
await this._deleteOrphanVdis()
this._oldEntries = mostRecentEntry !== undefined ? [mostRecentEntry] : []
}
}
@@ -400,7 +398,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
async cleanup() {
if (!this._settings.skipDeleteOldEntries) {
await this._deleteOldEntries()
await this._deleteOrphanVdis()
}
}
@@ -408,35 +405,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
return asyncMapSettled(this._oldEntries, vm => vm.$destroy({ bypassBlockedOperation: true }))
}
// VDIs from this job/VM/SR that ended up detached from any VM, because a previous run
// was interrupted or a storage backend refused to destroy a VDI at import time.
// Destroying a VM only reaches the VDIs still attached to it, _deleteOldEntries never cleans them.
async _deleteOrphanVdis() {
const sr = this._sr
const vmUuid = this._vmUuid
const jobId = this._job.id
const orphanVdis = sr.$VDIs.filter(
vdi =>
vdi?.managed &&
vdi.other_config[JOB_ID] === jobId &&
vdi.other_config[VM_UUID] === vmUuid &&
vdi.other_config[REPLICATED_TO_SR_UUID] === sr.uuid &&
vdi.type !== 'suspend' &&
// a VBD or VM missing from the cache (e.g. just destroyed) keeps the VDI, next run will handle it
vdi.$VBDs.every(vbd => vbd?.$VM?.is_control_domain === true)
)
await asyncEach(
orphanVdis,
vdi =>
sr.$xapi
.call('VDI.destroy', vdi.$ref)
.catch(error => Task.warning('failed to delete orphan VDI', { vdi: vdi.uuid, error })),
{ concurrency: 5 }
)
}
#decorateVmMetadata(backup, timestamp) {
const { _warmMigration } = this._settings
const sr = this._sr

View File

@@ -61,7 +61,7 @@ export class AbstractAggregatedXapiWriter {
setOldReplicaList() {
debug(`setOldReplicaList for vm ${this.#props.vmUuid}`)
const scheduleId = this.#props.schedule.id
const scheduleId = this.#props.scheduleId
const vmUuid = this.#props.vmUuid
const settings = this.#props.settings
@@ -80,35 +80,14 @@ export class AbstractAggregatedXapiWriter {
)
}
/**
* Deletes the old replicas computed by `setOldReplicaList`.
*
* @param {Object} [options]
* @param {boolean} [options.keepMostRecent=false] - Keep the most recent old replica, which is the base of the
* next incremental replication (needed when copyRetention is 1 with deleteFirst). The kept replica stays in
* the list, so a later call without `keepMostRecent` (from `cleanup()`) deletes it once the transfer is done.
*/
async deleteOldReplicas({ keepMostRecent = false } = {}) {
let toDelete = this.#oldVmReplicaList
let kept
// the list is sorted by datetime (oldest first), so the last entry is the most recent replica,
// which is the base of the next incremental replication
if (keepMostRecent && toDelete.length > 0) {
kept = toDelete[toDelete.length - 1]
toDelete = toDelete.slice(0, -1)
}
debug(`deleteOldReplicas will delete ${toDelete.length} replicated vms for vm ${this.#props.vmUuid}`)
async deleteOldReplicas() {
debug(`deleteOldReplicas will delete ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
// destroy the VM on the right xapi
await asyncMapSettled(toDelete, async vm => {
await asyncMapSettled(this.#oldVmReplicaList, async vm => {
debug('will delete old replica', vm.name_label)
await vm.$xapi.VM_destroy(vm.$ref)
})
debug(`deleteOldReplicas deleted ${toDelete.length} replicated vms for vm ${this.#props.vmUuid}`)
// whatever is left (the kept base, if any) will be deleted by the next call, from cleanup()
this.#oldVmReplicaList = kept !== undefined ? [kept] : []
debug(`deleteOldReplicas deleted ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
}
async setupWriters() {

View File

@@ -1,182 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
import { DATETIME, JOB_ID, REPLICATED_TO_SR_UUID, SCHEDULE_ID, VM_UUID } from '../../_otherConfig.mjs'
// This exercises setOldReplicaList()/deleteOldReplicas() through the REAL
// listReplicatedVms() (same fixture style as ./_listReplicatedVms.test.mjs), rather
// than mocking the module. This is deliberate: `node:test`'s module mocking needs
// --experimental-test-module-mocks, which this repo's `test` script (and therefore
// the pre-commit hook) does not pass, so a mocked version of this test would never
// pass CI. Going through the real filtering logic also means this test genuinely
// proves the fix — not just that a particular value gets forwarded to a spy.
const SOURCE_VM_UUID = 'vm-1'
const SR_UUID = 'sr-1'
const SCHEDULE_A = 'schedule-a'
const SCHEDULE_B = 'schedule-b'
const replicaTarget = { uuid: 'replica-target-uuid' } // stand-in for $snapshot_of on new-style replicas
/**
* Builds a fake "new-style" replication snapshot: a snapshot of the replicated
* target VM (not of the source VM), tagged with the schedule/job that produced it.
*/
const makeReplicaSnapshot = ({ id, datetime, scheduleId, srUuid = SR_UUID, vmUuid = SOURCE_VM_UUID }) => ({
$id: id,
$type: 'VM',
blocked_operations: {},
is_a_snapshot: true,
is_a_template: false,
name_label: id,
$snapshot_of: replicaTarget,
other_config: {
[DATETIME]: datetime,
[JOB_ID]: 'job-1',
[SCHEDULE_ID]: scheduleId,
[VM_UUID]: vmUuid,
[REPLICATED_TO_SR_UUID]: srUuid,
},
})
const makeXapi = vms => ({ objects: { indexes: { type: { VM: Object.fromEntries(vms.map(vm => [vm.$id, vm])) } } } })
/** Attaches $xapi/$ref (as deleteOldReplicas expects) and a VM_destroy spy to each vm. */
function wireForDestroy(vms, destroyCalls) {
const xapi = makeXapi(vms)
for (const vm of vms) {
vm.$xapi = { objects: xapi.objects, VM_destroy: ref => destroyCalls.push(ref) }
vm.$ref = vm.$id
}
return xapi
}
function makeSr(xapi, uuid = SR_UUID) {
return {
uuid,
name_label: `sr-${uuid}`,
physical_utilisation: 0,
physical_size: 100,
$xapi: xapi,
}
}
function makeWriter({ srs, vmUuid = SOURCE_VM_UUID, scheduleId, copyRetention }) {
return new AbstractAggregatedXapiWriter({
srs,
ReplicationWriter: class {},
vmUuid,
schedule: { id: scheduleId }, // <-- the real shape passed by the caller
settings: { copyRetention },
job: { settings: {} },
})
}
describe('AbstractAggregatedXapiWriter#setOldReplicaList / deleteOldReplicas', () => {
it('finds and prunes replicas for its own schedule down to copyRetention', async () => {
const replicas = [
makeReplicaSnapshot({ id: 'a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A }),
makeReplicaSnapshot({ id: 'a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A }),
makeReplicaSnapshot({ id: 'a-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A }),
makeReplicaSnapshot({ id: 'a-4', datetime: '20260104T000000Z', scheduleId: SCHEDULE_A }),
]
const destroyCalls = []
const xapi = wireForDestroy(replicas, destroyCalls)
const writer = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_A, copyRetention: 2 })
writer.setOldReplicaList()
await writer.deleteOldReplicas()
// copyRetention 2 keeps the running copy + 1 old one => 4 - 1 = 3 old ones pruned.
// (If setOldReplicaList still reads props.scheduleId instead of props.schedule.id,
// scheduleId is undefined, listReplicatedVms matches nothing, and destroyCalls
// stays empty — this assertion catches exactly that regression.)
assert.equal(
destroyCalls.length,
replicas.length - (2 - 1),
`expected ${replicas.length - 1} replicas pruned for retention 2, got ${destroyCalls.length}. ` +
'An empty destroyCalls list means listReplicatedVms was called with an undefined schedule id ' +
'and matched nothing (the scheduleId-vs-schedule.id bug).'
)
})
it('does not touch another schedule’s replicas on the same VM/SR when pruning its own', async () => {
const scheduleAReplicas = [
makeReplicaSnapshot({ id: 'a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A }),
makeReplicaSnapshot({ id: 'a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A }),
makeReplicaSnapshot({ id: 'a-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A }),
]
const scheduleBReplicas = [
makeReplicaSnapshot({ id: 'b-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_B }),
makeReplicaSnapshot({ id: 'b-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_B }),
makeReplicaSnapshot({ id: 'b-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_B }),
makeReplicaSnapshot({ id: 'b-4', datetime: '20260104T000000Z', scheduleId: SCHEDULE_B }),
makeReplicaSnapshot({ id: 'b-5', datetime: '20260105T000000Z', scheduleId: SCHEDULE_B }),
]
// Both schedules' replicas coexist in the same xapi, same SR, same source VM —
// exactly the real-world "multiple schedules on one job" case the bug breaks.
const allReplicas = [...scheduleAReplicas, ...scheduleBReplicas]
const destroyCalls = []
const xapi = wireForDestroy(allReplicas, destroyCalls)
// Prune schedule A only (retention 1: keep the running copy, drop the rest).
const writerA = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_A, copyRetention: 1 })
writerA.setOldReplicaList()
await writerA.deleteOldReplicas()
const destroyedIds = destroyCalls.slice().sort()
assert.deepEqual(
destroyedIds,
['a-1', 'a-2', 'a-3'],
'pruning schedule A should only ever destroy schedule A replicas, never schedule B’s — ' +
`got ${JSON.stringify(destroyedIds)}`
)
// Now prune schedule B (retention 2: keep the running copy + 1 old one => drop 4 of 5).
const writerB = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_B, copyRetention: 2 })
writerB.setOldReplicaList()
await writerB.deleteOldReplicas()
const destroyedAfterB = destroyCalls.slice(3).sort() // calls added since writerA's pass
assert.equal(destroyedAfterB.length, scheduleBReplicas.length - (2 - 1))
assert.ok(
destroyedAfterB.every(id => id.startsWith('b-')),
`pruning schedule B should only destroy schedule B replicas — got ${JSON.stringify(destroyedAfterB)}`
)
})
it('calls listReplicatedVms scoped to each storage repository in turn', async () => {
const srAReplicas = [
makeReplicaSnapshot({ id: 'sr-a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-a' }),
makeReplicaSnapshot({ id: 'sr-a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-a' }),
]
const srBReplicas = [
makeReplicaSnapshot({ id: 'sr-b-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
makeReplicaSnapshot({ id: 'sr-b-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
makeReplicaSnapshot({ id: 'sr-b-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
]
const destroyCalls = []
// Both SRs happen to share the same underlying xapi object index in this fixture
// (as they would if source and destination are on the same pool); what matters is
// that each SR's uuid scopes its own replicas.
const xapi = wireForDestroy([...srAReplicas, ...srBReplicas], destroyCalls)
const writer = makeWriter({
srs: [makeSr(xapi, 'sr-a'), makeSr(xapi, 'sr-b')],
scheduleId: SCHEDULE_A,
copyRetention: 1, // keep only the running copy on each SR
})
writer.setOldReplicaList()
await writer.deleteOldReplicas()
assert.deepEqual(
destroyCalls.slice().sort(),
['sr-a-1', 'sr-a-2', 'sr-b-1', 'sr-b-2', 'sr-b-3'],
'retention 1 should prune every replica on every configured SR down to none kept aside'
)
})
})

View File

@@ -1,18 +1,9 @@
import { AbstractWriter } from './_AbstractWriter.mjs'
export class AbstractFullWriter extends AbstractWriter {
async run({ checksum, maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
async run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
try {
return await this._run({
checksum,
maxStreamLength,
timestamp,
sizeContainer,
stream,
streamLength,
vm,
vmSnapshot,
})
return await this._run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot })
} finally {
// ensure stream is properly closed
stream.destroy()

View File

@@ -12,14 +12,11 @@ const { warn } = createLogger('xo:backups:vdiRestoreTargets')
*/
/**
* Attach the backup disk itself to the restored VM, through a live mount: no copy, read only —
* unless it is cached, which makes it read/write.
* Attach the backup disk itself to the restored VM, through a live mount: no copy, read only.
*
* @typedef {object} LiveMountVdiTarget
* @property {'live-mount'} type
* @property {string} host - uuid of the host the disk is attached to
* @property {string} [cacheSr] - uuid of the SR of the local VDI the disk is materialized into, which
* also holds the writes, lost on unmount
*/
/**
@@ -136,20 +133,6 @@ export function hasLiveMountTarget(mapVdisSrs) {
return Object.values(mapVdisSrs ?? {}).some(value => value?.type === 'live-mount')
}
/**
* Whether a raw `mapVdisSrs` setting asks for at least one cached, hence read/write, live mount.
*
* Such a mount needs the VM of the appliance serving it, which the caller has to look up. A `null`
* cache SR does not ask for one: it is invalid, and left to {@link normalizeVdiRestoreTargets} to
* reject, rather than failing on the lookup of a VM which was never needed.
*
* @param {object} [mapVdisSrs]
* @returns {boolean}
*/
export function hasCachedLiveMountTarget(mapVdisSrs) {
return Object.values(mapVdisSrs ?? {}).some(value => value?.type === 'live-mount' && value.cacheSr != null)
}
/**
* Normalize the `mapVdisSrs` restore setting into per-VDI targets.
*
@@ -203,17 +186,10 @@ function normalizeTarget(vdiUuid, value, useDifferentialRestore) {
}
if (type === 'live-mount') {
const { cacheSr, host } = value
if (!isNonEmptyString(host)) {
if (!isNonEmptyString(value.host)) {
throw invalidTarget(vdiUuid, 'a live mount requires the id of the host the disk is attached to')
}
if (cacheSr === undefined) {
return { type: 'live-mount', host }
}
if (!isNonEmptyString(cacheSr)) {
throw invalidTarget(vdiUuid, `expected the uuid of the cache SR, got ${typeof cacheSr}`)
}
return { type: 'live-mount', host, cacheSr }
return { type: 'live-mount', host: value.host }
}
if (type === 'restore') {

View File

@@ -1,7 +1,7 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { hasCachedLiveMountTarget, hasLiveMountTarget, normalizeVdiRestoreTargets } from './_vdiRestoreTargets.mjs'
import { hasLiveMountTarget, normalizeVdiRestoreTargets } from './_vdiRestoreTargets.mjs'
const VDI_1 = 'vdi-uuid-1'
const VDI_2 = 'vdi-uuid-2'
@@ -64,11 +64,6 @@ describe('normalizeVdiRestoreTargets()', () => {
assert.equal(targets.getLiveMountHost(), HOST)
})
it('keeps the cache SR of a live mount', () => {
const targets = normalizeVdiRestoreTargets({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: SR } })
assert.deepEqual(targets.get(VDI_1), { type: 'live-mount', host: HOST, cacheSr: SR })
})
it('accepts a restore without SR', () => {
const targets = normalizeVdiRestoreTargets({ [VDI_1]: { type: 'restore' } })
assert.deepEqual(targets.get(VDI_1), { type: 'restore', sr: undefined, useDifferential: false })
@@ -102,13 +97,6 @@ describe('normalizeVdiRestoreTargets()', () => {
)
})
it('rejects a non string cache SR', () => {
assert.throws(
() => normalizeVdiRestoreTargets({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: null } }),
/expected the uuid of the cache SR/
)
})
it('rejects a non string SR', () => {
assert.throws(() => normalizeVdiRestoreTargets({ [VDI_1]: { type: 'restore', sr: 42 } }), /expected an SR uuid/)
})
@@ -142,20 +130,3 @@ describe('hasLiveMountTarget()', () => {
assert.equal(hasLiveMountTarget({ [VDI_1]: SR, [VDI_2]: null, [VDI_3]: { type: 'ignore' } }), false)
})
})
describe('hasCachedLiveMountTarget()', () => {
it('detects a cached live mount entry', () => {
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: SR } }), true)
})
it('is false for an uncached live mount, and every other shape', () => {
assert.equal(hasCachedLiveMountTarget(undefined), false)
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST } }), false)
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: SR, [VDI_2]: null, [VDI_3]: { type: 'ignore' } }), false)
})
// the normalization rejects it, which is the error worth reporting
it('is false for a null cache SR', () => {
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: null } }), false)
})
})

View File

@@ -1,26 +0,0 @@
import { isExpression, compileExpression } from './_expressionPredicate.mjs'
import { invalidParameters } from 'xo-common/api-errors.js'
export function assertValidSettingsExpression(settings, keys) {
for (const [scheduleId, scheduleSettings] of Object.entries(settings ?? {})) {
for (const key of keys) {
assertValidExpression(`settings/${scheduleId === '' ? 'general' : scheduleId}/${key}`, scheduleSettings?.[key])
}
}
}
export function assertValidExpression(name, value) {
if (isExpression(value)) {
try {
compileExpression(value)
} catch (error) {
throw invalidParameters(`Invalid ${name} expression: ${error.message}`, [
{ instancePath: `/${name}`, message: error.message },
])
}
} else if (typeof value === 'string' && value.trim() === '') {
throw invalidParameters(`Invalid ${name} expression: must be a non-empty expression`, [
{ instancePath: `/${name}`, message: 'must be a non-empty expression' },
])
}
}

View File

@@ -1,77 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { invalidParameters } from 'xo-common/api-errors.js'
import { assertValidExpression, assertValidSettingsExpression } from './backupExpressions.mjs'
const VALID = 'vm:tags:prod'
const INVALID = 'vm:('
// checks that the error is `invalidParameters` and points to the expected field
const isInvalidAt = instancePath => error =>
invalidParameters.is(error) && error.data.errors[0].instancePath === instancePath
describe('assertValidExpression', function () {
it('accepts a valid expression', function () {
assert.doesNotThrow(() => assertValidExpression('srs', VALID))
})
it('rejects an invalid expression with invalidParameters', function () {
assert.throws(() => assertValidExpression('srs', INVALID), isInvalidAt('/srs'))
})
it('ignores values which are not expressions', function () {
for (const value of [undefined, '3', { id: 'sr-1' }, { id: { __or: ['sr-1', 'sr-2'] } }]) {
assert.doesNotThrow(() => assertValidExpression('srs', value))
}
})
it('rejects blank strings', function () {
for (const value of ['', ' ']) {
assert.throws(() => assertValidExpression('srs', value), isInvalidAt('/srs'))
}
})
})
describe('assertValidSettingsExpression', function () {
const KEYS = ['healthCheckVmsWithTags', 'fullInterval']
it('accepts valid expressions in the general and the schedule settings', function () {
assert.doesNotThrow(() =>
assertValidSettingsExpression(
{ '': { fullInterval: VALID }, 'schedule-1': { healthCheckVmsWithTags: VALID } },
KEYS
)
)
})
it('rejects an invalid expression in the schedule settings and points to it', function () {
assert.throws(
() => assertValidSettingsExpression({ '': {}, 'schedule-1': { fullInterval: INVALID } }, KEYS),
isInvalidAt('/settings/schedule-1/fullInterval')
)
})
it('rejects an invalid expression in the general settings', function () {
// the general settings are keyed by '', named "general" in the path.
assert.throws(
() => assertValidSettingsExpression({ '': { healthCheckVmsWithTags: INVALID } }, KEYS),
isInvalidAt('/settings/general/healthCheckVmsWithTags')
)
})
it('ignores the legacy array and number values', function () {
assert.doesNotThrow(() =>
assertValidSettingsExpression({ 'schedule-1': { healthCheckVmsWithTags: ['tag'], fullInterval: 3 } }, KEYS)
)
})
it('ignores the keys it has not been asked to check', function () {
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': { vmFilter: INVALID } }, KEYS))
})
it('handles missing settings and undefined entries', function () {
assert.doesNotThrow(() => assertValidSettingsExpression(undefined, KEYS))
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': undefined }, KEYS))
})
})

View File

@@ -43,7 +43,7 @@ A VM directory holds **both** the full and the incremental backups of that VM: t
Transient entries, not listed in the trees below:
- `xo-vm-backups/<VM UUID>.lock`, held while a job writes to that VM's folder and while `cleanVm` cleans or merges it (after a job, after a deletion, or in the merge worker on local remotes). It is effective on S3 (an object rewritten every 30 seconds) and Azure (blob lease) too. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
- `xo-vm-backups/<VM UUID>.lock`, held for the duration of a job run on that VM. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
- `xo-vm-backups/.queue/clean-vm/<YYYYMMDD>T<HHmmss>-<random>`, the merge worker queue
- `.<VHD file name>.merge.json`, in a VDI directory, interrupted merge states.
@@ -167,7 +167,6 @@ Append-only log of what happened to the VM backup metadata.
- entries are never modified nor overwritten, so it works on an immutable remote
- written unencrypted, so it stays readable without the encryption key
- one directory per day: a reader lists only the days it misses, a purge drops whole directories
- xo-server removes the days older than `backups.journalRetention` from every enabled remote when it starts, then once a day; it always keeps the `fullListingInterval` of the remote, during which it reads the journal back; a day which cannot be removed is skipped, and the next purge tries it again
- the time part has a fixed width, so sorting the names by name sorts them by date
- `event` is `add`, `change` or `del`, and the file content also has a `reason` field
- the name of the file is only there to be read by a human: XO reads every field from the file content

View File

@@ -1,28 +0,0 @@
import { formatFilenameDate } from './_filenameDate.mjs'
/**
* XAPI names of what a live mount of a backed up disk shows the user, so the ones created by the REST
* API and by a restore read the same.
*
* The VDI attached to the host keeps the name of the backed up disk: that is the disk of the VM as
* the user knows it. The SR says what it is, and which restore point it serves, the same way as a
* restored VM is named.
*
* @param {object} params
* @param {string} params.vmNameLabel - name of the backed up VM
* @param {string} params.vdiNameLabel - name of the backed up disk
* @param {number} params.timestamp - of the restore point, in ms
* @param {boolean} [params.readWrite] - whether the mount is a live restore rather than a live mount
* @returns {{ srNameLabel: string, vdiNameLabel: string, vdiNameDescription: string }} - the `xapiLabels` of the live mount mixin
*/
export function liveMountXapiLabels({ vmNameLabel, vdiNameLabel, timestamp, readWrite = false }) {
const what = readWrite ? 'live restore' : 'live mount'
const restorePoint = `${vmNameLabel} (${formatFilenameDate(new Date(timestamp))})`
return {
srNameLabel: `[XO ${what}] ${restorePoint}`,
vdiNameLabel,
vdiNameDescription: readWrite
? `Live restore of ${restorePoint}: what is written to it is lost on unmount.`
: `Read-only live mount of ${restorePoint}, removed on unmount.`,
}
}

View File

@@ -1,27 +0,0 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { liveMountXapiLabels } from './liveMountXapiLabels.mjs'
const TIMESTAMP = 1754035712000 // 2025-08-01T08:08:32Z
describe('liveMountXapiLabels', () => {
it('keeps the name of the disk and names the SR after the restore point', () => {
assert.deepEqual(liveMountXapiLabels({ timestamp: TIMESTAMP, vdiNameLabel: 'system', vmNameLabel: 'web01' }), {
srNameLabel: '[XO live mount] web01 (20250801T080832Z)',
vdiNameLabel: 'system',
vdiNameDescription: 'Read-only live mount of web01 (20250801T080832Z), removed on unmount.',
})
})
it('says a live restore loses what is written to it', () => {
const xapiLabels = liveMountXapiLabels({
readWrite: true,
timestamp: TIMESTAMP,
vdiNameLabel: 'system',
vmNameLabel: 'web01',
})
assert.equal(xapiLabels.srNameLabel, '[XO live restore] web01 (20250801T080832Z)')
assert.match(xapiLabels.vdiNameDescription, /lost on unmount/)
})
})

View File

@@ -8,7 +8,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.78.0",
"version": "0.76.0",
"engines": {
"node": ">=14.18"
},
@@ -27,20 +27,19 @@
"@vates/disposable": "^0.1.6",
"@vates/fuse-vhd": "^2.1.3",
"@vates/generator-toolbox": "^1.1.3",
"@vates/nbd-client": "^3.7.0",
"@vates/nbd-client": "^3.6.0",
"@vates/parse-duration": "^0.1.1",
"@vates/task": "^0.7.0",
"@vates/types": "^2.5.0",
"@vates/types": "^2.3.0",
"@xen-orchestra/async-map": "^0.1.5",
"@xen-orchestra/disk-transform": "^1.3.4",
"@xen-orchestra/fs": "^4.10.0",
"@xen-orchestra/disk-transform": "^1.3.3",
"@xen-orchestra/fs": "^4.9.5",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/qcow2": "^1.4.1",
"@xen-orchestra/qcow2": "^1.4.0",
"@xen-orchestra/template": "^0.1.1",
"@xen-orchestra/backup-archive": "^1.2.0",
"@xen-orchestra/backup-archive": "^1.1.3",
"app-conf": "^3.0.0",
"compare-versions": "^6.0.0",
"complex-matcher": "^1.1.1",
"d3-time-format": "^4.1.0",
"decorator-synchronized": "^0.6.0",
"golike-defer": "^0.5.1",
@@ -56,10 +55,8 @@
"tar": "^7.5.19",
"uuid": "^9.0.0",
"value-matcher": "^0.2.0",
"vhd-lib": "^4.17.0",
"xen-api": "^5.1.0",
"xo-common": "^0.11.0",
"xo-remote-parser": "^2.0.0",
"vhd-lib": "^4.16.2",
"xen-api": "^5.0.1",
"yazl": "^2.5.1"
},
"devDependencies": {
@@ -70,7 +67,7 @@
"typescript": "^5.9.3"
},
"peerDependencies": {
"@xen-orchestra/xapi": "^9.1.0"
"@xen-orchestra/xapi": "^9.0.1"
},
"license": "AGPL-3.0-or-later",
"author": {

View File

@@ -18,7 +18,7 @@
"preferGlobal": true,
"dependencies": {
"golike-defer": "^0.5.1",
"xen-api": "^5.1.0"
"xen-api": "^5.0.1"
},
"scripts": {
"postversion": "npm publish"

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/disk-cli",
"version": "2.2.5",
"version": "2.2.3",
"description": "CLI tool to inspect and manage disks in backup repositories",
"type": "module",
"bin": {
@@ -20,13 +20,13 @@
},
"dependencies": {
"@vates/async-each": "^1.0.3",
"@xen-orchestra/backups": "^0.78.0",
"@xen-orchestra/backup-archive": "^1.2.0",
"@xen-orchestra/disk-transform": "^1.3.4",
"@xen-orchestra/fs": "^4.10.0",
"@xen-orchestra/qcow2": "^1.4.1",
"@xen-orchestra/backups": "^0.76.0",
"@xen-orchestra/backup-archive": "^1.1.3",
"@xen-orchestra/disk-transform": "^1.3.3",
"@xen-orchestra/fs": "^4.9.5",
"@xen-orchestra/qcow2": "^1.4.0",
"promise-toolbox": "^0.21.0",
"vhd-lib": "^4.17.0"
"vhd-lib": "^4.16.2"
},
"devDependencies": {
"@types/node": "^22.3.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@xen-orchestra/disk-transform",
"version": "1.3.4",
"version": "1.3.3",
"main": "./dist/index.mjs",
"license": "MIT",
"private": false,

View File

@@ -9,7 +9,7 @@ async function consumeGenerator(source: AsyncGenerator) {
return 'finished'
}
test('Disk class', async t => {
const testRes = await t.test('synchronized fork disk ', async () => {
const testRes = await t.test('synchronised fork disk ', async () => {
const blockSize = 1024
const nbBlocks = 20
const source = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })

View File

@@ -12,11 +12,11 @@ export class ThrottledDisk extends DiskPassthrough {
const generator = await this.source.buildDiskBlockGenerator()
//throttle want to be able to know the length of the data
async function* generatorWithLength() {
for await (const block of generator) {
// keep the other properties of the block (like the compression of a raw VHD directory block)
for await (const { index, data } of generator) {
yield {
...block,
length: block.data.length,
index,
data,
length: data.length,
}
}
}

View File

@@ -1,32 +0,0 @@
import { test } from 'node:test'
import assert from 'node:assert'
import { Throttle } from '@vates/generator-toolbox'
import { DiskBlock } from './Disk.mjs'
import { DebugDisk } from './utils/DebugDisk.mjs'
import { ThrottledDisk } from './Throttled.mjs'
type TaggedDiskBlock = DiskBlock & { compression: string }
// a disk yielding blocks with an additional property, like the raw blocks of a VHD directory
class TaggedDebugDisk extends DebugDisk {
async readBlock(index: number): Promise<TaggedDiskBlock> {
return { ...(await super.readBlock(index)), compression: 'brotli' }
}
}
test('ThrottledDisk keeps the other properties of the blocks', async () => {
const blockSize = 1024
const nbBlocks = 5
const source = new TaggedDebugDisk({ nbBlocks, blockSize, fillRate: 100 })
await source.init()
// speed 0 disables the throttling
const disk = new ThrottledDisk(source, new Throttle(0))
let nbBlocksRead = 0
for await (const block of disk.diskBlocks()) {
assert.strictEqual((block as TaggedDiskBlock).compression, 'brotli')
assert.strictEqual(block.data.length, blockSize)
nbBlocksRead++
}
assert.strictEqual(nbBlocksRead, nbBlocks)
})

View File

@@ -18,7 +18,7 @@ ee.on('start', async function () {
// listeners have settled
await ee.emitAsync('start')
// by default, it will rejects as soon as one listener reject, you can customize
// by default, it will rejects as soon as one listener reject, you can customise
// error handling though:
await ee.emitAsync(
{

View File

@@ -36,7 +36,7 @@ ee.on('start', async function () {
// listeners have settled
await ee.emitAsync('start')
// by default, it will rejects as soon as one listener reject, you can customize
// by default, it will rejects as soon as one listener reject, you can customise
// error handling though:
await ee.emitAsync(
{

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/fs",
"version": "4.10.0",
"version": "4.9.5",
"license": "AGPL-3.0-or-later",
"description": "The File System for Xen Orchestra backups.",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",

View File

@@ -28,7 +28,6 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
ivLength: 0,
authTagLength: 0,
encryptData: buffer => buffer,
encryptDataParts: buffers => buffers,
encryptStream: stream => stream,
decryptData: buffer => buffer,
decryptStream: stream => stream,
@@ -110,17 +109,11 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
)
}
// data can be an array of buffers, encrypted as their concatenation
// returns the encrypted file as an array of buffers, to be written one after the other
function encryptDataParts(data) {
function encryptData(buffer) {
const iv = crypto.randomBytes(ivLength)
const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
const encrypted = Array.isArray(data) ? data.map(buffer => cipher.update(buffer)) : [cipher.update(data)]
return [iv, ...encrypted, cipher.final(), authTagLength > 0 ? cipher.getAuthTag() : Buffer.alloc(0)]
}
function encryptData(data) {
return Buffer.concat(encryptDataParts(data))
const encrypted = cipher.update(buffer)
return Buffer.concat([iv, encrypted, cipher.final(), authTagLength > 0 ? cipher.getAuthTag() : Buffer.alloc(0)])
}
function decryptData(buffer) {
@@ -135,10 +128,7 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
encrypted = buffer.slice(ivLength)
}
const decrypted = decipher.update(encrypted)
// final() must always be called: it checks the auth tag
const final = decipher.final()
// nothing is left with stream modes (like GCM): no need to copy the data
return final.length === 0 ? decrypted : Buffer.concat([decrypted, final])
return Buffer.concat([decrypted, decipher.final()])
}
return {
@@ -148,7 +138,6 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
authTagLength,
ivLength,
encryptData,
encryptDataParts,
encryptStream,
decryptData,
decryptStream,

View File

@@ -36,16 +36,6 @@ algorithms.forEach(algorithm => {
assert.equal(decrypted.equals(buffer), true)
})
it('handle an array of buffers', () => {
// like a VHD block: a bitmap then the data, sizes not aligned on the cipher block size
const parts = [crypto.randomBytes(512), crypto.randomBytes(1024 * 1024 + 1)]
const encrypted = Buffer.concat(encryptor.encryptDataParts(parts))
// same layout as a single buffer: existing remotes must still be readable
const decrypted = encryptor.decryptData(encrypted)
assert.equal(decrypted.equals(Buffer.concat(parts)), true)
})
it('handle stream', async () => {
const stream = Readable.from(buffer)
stream.length = buffer.length

Some files were not shown because too many files have changed in this diff Show More