mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-20 05:48:38 -06:00
Compare commits
4 Commits
docs/body-
...
fix_sync_b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b69b339a6 | ||
|
|
19b1fe1009 | ||
|
|
86b9686c5c | ||
|
|
0eab596880 |
22
.github/workflows/vectura.yml
vendored
22
.github/workflows/vectura.yml
vendored
@@ -1,19 +1,8 @@
|
||||
name: Vectura
|
||||
|
||||
# only when vectura changes: the rest of the repository does not build it
|
||||
#
|
||||
# pull requests rather than every push: GitHub filters a push on the diff with
|
||||
# the previous push, which after a rebase on `master` (or a merge of it)
|
||||
# contains the vectura changes of `master`, whereas it filters a pull request
|
||||
# on the diff with the merge base, i.e. the changes of the branch only
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '@xen-orchestra/vmware-explorer/vectura/**'
|
||||
- .github/workflows/vectura.yml
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- '@xen-orchestra/vmware-explorer/vectura/**'
|
||||
- .github/workflows/vectura.yml
|
||||
@@ -45,8 +34,8 @@ jobs:
|
||||
- name: Hold the coverage floor
|
||||
run: cargo test --locked --test coverage -- --ignored
|
||||
|
||||
# builds the binary as release.sh does, checks it, then checks the one in git,
|
||||
# which xo-server runs, is that build
|
||||
# builds the binary as a release does, then checks it and the one in git,
|
||||
# which xo-server runs
|
||||
binary:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -65,9 +54,4 @@ jobs:
|
||||
|
||||
- run: scripts/build.sh "$RUNNER_TEMP/build"
|
||||
- run: scripts/check-binary.sh "$RUNNER_TEMP/build/vectura"
|
||||
- name: The binary in git is the build of these sources
|
||||
run: |
|
||||
cmp "$RUNNER_TEMP/build/vectura" vectura || {
|
||||
echo '::error::vectura is not the build of these sources: run scripts/build.sh . in vectura/ and commit the binary'
|
||||
exit 1
|
||||
}
|
||||
- run: scripts/check-binary.sh vectura
|
||||
|
||||
@@ -19,12 +19,12 @@
|
||||
},
|
||||
"main": "./index.mjs",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/fs": "^4.10.0",
|
||||
"@xen-orchestra/fs": "^4.9.5",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"fuse-native": "^2.2.6",
|
||||
"lru-cache": "^7.14.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"vhd-lib": "^4.17.0"
|
||||
"vhd-lib": "^4.16.2"
|
||||
},
|
||||
"bin": {
|
||||
"xo-fuse-vhd": "./cli.mjs"
|
||||
|
||||
@@ -15,7 +15,7 @@ The target drives login negotiation onto a single, fixed code path:
|
||||
It implements SendTargets discovery, login (`AuthMethod=None` or one-way
|
||||
**CHAP** — see [Authentication](#authentication-chap)), the ~10 SCSI
|
||||
commands a Linux initiator issues to attach and use a block device
|
||||
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`/`0xb0`, `REPORT LUNS`, `READ CAPACITY 10/16`,
|
||||
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`, `REPORT LUNS`, `READ CAPACITY 10/16`,
|
||||
`READ`/`WRITE 10/16`, `TEST UNIT READY`, `REQUEST SENSE`, `MODE SENSE`,
|
||||
`SYNCHRONIZE CACHE`), the full R2T / Data-Out write path, NOP keepalive replies,
|
||||
and Logout. Commands may be outstanding concurrently (Data-Out PDUs are routed
|
||||
@@ -28,19 +28,6 @@ The package also ships a userspace **initiator** (`IscsiInitiator`) and an
|
||||
Not implemented (yet): mutual (two-way) CHAP, Task Management (abort/reset),
|
||||
target-initiated NOP keepalives, multiple connections/LUNs.
|
||||
|
||||
What a peer can make the target do is bounded:
|
||||
|
||||
- connections that have not logged in yet are limited in time (`loginTimeoutMs`)
|
||||
and number (`maxPendingConnections`), and `allowedInitiatorAddresses` (the
|
||||
source addresses of the expected initiators) closes the other ones as soon as
|
||||
they are accepted, reporting them to `onConnectionRefused`
|
||||
- a PDU data segment is at most 8 KiB during login, then the declared
|
||||
`MaxRecvDataSegmentLength` (256 KiB), checked before it is read
|
||||
- a single `READ`/`WRITE` is at most `maxTransferLength` (8 MiB by default),
|
||||
advertised in the block limits VPD page so that initiators split their I/O
|
||||
- a `BlockDevice` with `readOnly: true` is reported write-protected, and `WRITE`
|
||||
is refused with `DATA PROTECT` before any data is solicited
|
||||
|
||||
### CLI
|
||||
|
||||
The package ships a `vates-iscsi` binary that serves a file as a LUN:
|
||||
@@ -152,8 +139,6 @@ class RandomAccessDiskLun {
|
||||
}
|
||||
return out
|
||||
}
|
||||
// reported write-protected: WRITE is refused before reaching write()
|
||||
readOnly = true
|
||||
write() {
|
||||
throw new Error('read-only LUN')
|
||||
}
|
||||
|
||||
@@ -18,14 +18,14 @@ The package has two independent roles with very different maturity:
|
||||
|
||||
## Session and connection
|
||||
|
||||
| Aspect | Target | Initiator |
|
||||
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------- |
|
||||
| Connections per session | `MaxConnections=1`, enforced | opens exactly one |
|
||||
| Session reinstatement | **no** — no ISID tracking; once a new connection has established a Normal session (CHAP included), it tears down and replaces whatever connection currently exists, rather than RFC 7143's ISID-matched reinstatement (see `index.mts`'s class doc for the reasoning: safe here because the target is ephemeral, single-consumer, and CHAP-guarded). Until then it does not affect the current one, and is bounded by `loginTimeoutMs` and `maxPendingConnections` | n/a (single connection) |
|
||||
| Multiple LUNs | **no** — exactly one LUN, always LUN 0 | reads are hardcoded to LUN 0 |
|
||||
| `ErrorRecoveryLevel` | `0` only | requests `0` |
|
||||
| Header/data digests | `HeaderDigest=None`, `DataDigest=None` — no CRC32C on the wire | same |
|
||||
| Markers (`IFMarker`/`OFMarker`) | not offered (RFC-deprecated) | not offered |
|
||||
| Aspect | Target | Initiator |
|
||||
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------- |
|
||||
| Connections per session | `MaxConnections=1`, enforced | opens exactly one |
|
||||
| Session reinstatement | **no** — no ISID tracking; a new connection always tears down and replaces whatever connection currently exists, rather than RFC 7143's ISID-matched reinstatement (see `index.mts`'s class doc for the reasoning: safe here because the target is ephemeral, single-consumer, and CHAP-guarded) | n/a (single connection) |
|
||||
| Multiple LUNs | **no** — exactly one LUN, always LUN 0 | reads are hardcoded to LUN 0 |
|
||||
| `ErrorRecoveryLevel` | `0` only | requests `0` |
|
||||
| Header/data digests | `HeaderDigest=None`, `DataDigest=None` — no CRC32C on the wire | same |
|
||||
| Markers (`IFMarker`/`OFMarker`) | not offered (RFC-deprecated) | not offered |
|
||||
|
||||
## Login / text negotiation
|
||||
|
||||
@@ -72,17 +72,17 @@ keys if it declines transit on the first exchange.
|
||||
Decoded explicitly (`types.mts#decodeCdb`); anything else maps to `unsupported` → `CHECK CONDITION` /
|
||||
`ILLEGAL_REQUEST` / `INVALID COMMAND OPERATION CODE`:
|
||||
|
||||
| Command | Variants |
|
||||
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `INQUIRY` | standard + EVPD `0x00`/`0x80`/`0x83` (vendor/product/revision/serial from `identity`) and `0xb0` (block limits: `maxTransferLength`) |
|
||||
| `REPORT LUNS` | |
|
||||
| `READ CAPACITY` | `(10)` and `(16)` |
|
||||
| `READ` | `(10)` and `(16)`; longer than `maxTransferLength` → `INVALID FIELD IN CDB` |
|
||||
| `WRITE` | `(10)` and `(16)`; longer than `maxTransferLength` → `INVALID FIELD IN CDB`; on a `readOnly` LUN → `DATA PROTECT` / `WRITE PROTECTED`, without soliciting data |
|
||||
| `TEST UNIT READY` | |
|
||||
| `REQUEST SENSE` | |
|
||||
| `MODE SENSE` | `(6)` and `(10)`; `WP` set on a `readOnly` LUN |
|
||||
| `SYNCHRONIZE CACHE` | `(10)` and `(16)` — flushes the backing `BlockDevice` |
|
||||
| Command | Variants |
|
||||
| ------------------- | ---------------------------------------------------------------- |
|
||||
| `INQUIRY` | standard + EVPD (vendor/product/revision/serial from `identity`) |
|
||||
| `REPORT LUNS` | |
|
||||
| `READ CAPACITY` | `(10)` and `(16)` |
|
||||
| `READ` | `(10)` and `(16)` |
|
||||
| `WRITE` | `(10)` and `(16)` |
|
||||
| `TEST UNIT READY` | |
|
||||
| `REQUEST SENSE` | |
|
||||
| `MODE SENSE` | `(6)` and `(10)` |
|
||||
| `SYNCHRONIZE CACHE` | `(10)` and `(16)` — flushes the backing `BlockDevice` |
|
||||
|
||||
**Not implemented**: persistent reservations, VAAI/thin-provisioning primitives (`UNMAP`, `WRITE SAME`, `COMPARE
|
||||
AND WRITE`, ...), any vendor-specific commands — anything beyond the ~10 commands a Linux initiator actually
|
||||
|
||||
@@ -33,7 +33,7 @@ The target drives login negotiation onto a single, fixed code path:
|
||||
It implements SendTargets discovery, login (`AuthMethod=None` or one-way
|
||||
**CHAP** — see [Authentication](#authentication-chap)), the ~10 SCSI
|
||||
commands a Linux initiator issues to attach and use a block device
|
||||
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`/`0xb0`, `REPORT LUNS`, `READ CAPACITY 10/16`,
|
||||
(`INQUIRY` + VPD `0x00`/`0x80`/`0x83`, `REPORT LUNS`, `READ CAPACITY 10/16`,
|
||||
`READ`/`WRITE 10/16`, `TEST UNIT READY`, `REQUEST SENSE`, `MODE SENSE`,
|
||||
`SYNCHRONIZE CACHE`), the full R2T / Data-Out write path, NOP keepalive replies,
|
||||
and Logout. Commands may be outstanding concurrently (Data-Out PDUs are routed
|
||||
@@ -46,19 +46,6 @@ The package also ships a userspace **initiator** (`IscsiInitiator`) and an
|
||||
Not implemented (yet): mutual (two-way) CHAP, Task Management (abort/reset),
|
||||
target-initiated NOP keepalives, multiple connections/LUNs.
|
||||
|
||||
What a peer can make the target do is bounded:
|
||||
|
||||
- connections that have not logged in yet are limited in time (`loginTimeoutMs`)
|
||||
and number (`maxPendingConnections`), and `allowedInitiatorAddresses` (the
|
||||
source addresses of the expected initiators) closes the other ones as soon as
|
||||
they are accepted, reporting them to `onConnectionRefused`
|
||||
- a PDU data segment is at most 8 KiB during login, then the declared
|
||||
`MaxRecvDataSegmentLength` (256 KiB), checked before it is read
|
||||
- a single `READ`/`WRITE` is at most `maxTransferLength` (8 MiB by default),
|
||||
advertised in the block limits VPD page so that initiators split their I/O
|
||||
- a `BlockDevice` with `readOnly: true` is reported write-protected, and `WRITE`
|
||||
is refused with `DATA PROTECT` before any data is solicited
|
||||
|
||||
### CLI
|
||||
|
||||
The package ships a `vates-iscsi` binary that serves a file as a LUN:
|
||||
@@ -170,8 +157,6 @@ class RandomAccessDiskLun {
|
||||
}
|
||||
return out
|
||||
}
|
||||
// reported write-protected: WRITE is refused before reaching write()
|
||||
readOnly = true
|
||||
write() {
|
||||
throw new Error('read-only LUN')
|
||||
}
|
||||
|
||||
@@ -35,11 +35,6 @@ VM boot mechanics, and deployment specifics live elsewhere and aren't reproduced
|
||||
- Re-verified end-to-end after deploying the fix: same S3-backed boot scenario completed successfully
|
||||
(~5 min total, vs ~20-25s for the equivalent local-storage boot — S3 itself is slow, but the mount no
|
||||
longer wedges).
|
||||
- Refined afterwards: the replacement used to happen as soon as the TCP connection was accepted, so any
|
||||
peer reaching the port (a port scan, a failed CHAP login, a Discovery session) interrupted the session in
|
||||
use. It now only happens once the new connection has established a Normal session; until then,
|
||||
connections are bounded by `loginTimeoutMs` and `maxPendingConnections`. Regression tests:
|
||||
`loopback.integ.mts`, `describe('connection admission', ...)`.
|
||||
|
||||
### Independently landed, cross-referenced against this pass's coverage
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@vates/iscsi",
|
||||
"version": "0.4.0",
|
||||
"version": "0.2.0",
|
||||
"main": "dist/index.mjs",
|
||||
"types": "dist/index.d.mts",
|
||||
"bin": {
|
||||
@@ -23,7 +23,7 @@
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@vates/read-chunk": "^1.2.1",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4",
|
||||
"@xen-orchestra/disk-transform": "^1.3.0",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"promise-toolbox": "^0.21.0"
|
||||
},
|
||||
|
||||
@@ -102,10 +102,10 @@ export class DiskBlockDevice implements BlockDevice {
|
||||
return result
|
||||
}
|
||||
|
||||
// the target refuses WRITE commands before they reach `write()`
|
||||
readonly readOnly = true
|
||||
|
||||
/** Always throws: the source disk is read-only. */
|
||||
/**
|
||||
* Always throws: the source disk is read-only. The target answers CHECK
|
||||
* CONDITION / MEDIUM ERROR, which initiators surface as an I/O error.
|
||||
*/
|
||||
async write(): Promise<void> {
|
||||
throw new Error('DiskBlockDevice is read-only')
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFile, execFileSync } from 'node:child_process'
|
||||
import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
|
||||
import { FileBlockDevice } from './index.mjs'
|
||||
|
||||
const execFileP = promisify(execFile)
|
||||
|
||||
const BLOCK_SIZE = 512
|
||||
const SIZE = 4 * 1024 * 1024 // 4 MiB
|
||||
|
||||
/**
|
||||
* Why this cannot run here, or `false` if it can. A real block device is the
|
||||
* whole point, and the cheapest one to make is a loop device — which needs root
|
||||
* and `losetup`.
|
||||
*/
|
||||
function skipReason(): string | false {
|
||||
if (process.getuid === undefined || process.getuid() !== 0) {
|
||||
return 'requires root to attach a loop device'
|
||||
}
|
||||
try {
|
||||
execFileSync('losetup', ['--version'], { stdio: 'ignore' })
|
||||
} catch {
|
||||
return 'requires util-linux (losetup not found)'
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const bytePattern = (i: number) => (i * 31 + 11) & 0xff
|
||||
|
||||
describe('FileBlockDevice on a loop device', { skip: skipReason() }, () => {
|
||||
let dir: string
|
||||
let devicePath: string
|
||||
|
||||
before(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-loop-'))
|
||||
const backing = join(dir, 'backing.img')
|
||||
await writeFile(backing, '')
|
||||
await truncate(backing, SIZE)
|
||||
const { stdout } = await execFileP('losetup', ['--find', '--show', backing])
|
||||
devicePath = stdout.trim()
|
||||
})
|
||||
|
||||
after(async () => {
|
||||
if (devicePath !== undefined) {
|
||||
await execFileP('losetup', ['--detach', devicePath])
|
||||
}
|
||||
await rm(dir, { force: true, recursive: true })
|
||||
})
|
||||
|
||||
const withDevice = async (fn: (device: FileBlockDevice) => Promise<void>): Promise<void> => {
|
||||
const device = new FileBlockDevice({ path: devicePath, size: SIZE })
|
||||
await device.open()
|
||||
try {
|
||||
await fn(device)
|
||||
} finally {
|
||||
await device.close()
|
||||
}
|
||||
}
|
||||
|
||||
it('accepts the device node and reports the capacity it was given', async () => {
|
||||
await withDevice(async device => {
|
||||
assert.equal(device.getSize(), SIZE)
|
||||
assert.equal(device.getBlockSize(), BLOCK_SIZE)
|
||||
})
|
||||
})
|
||||
|
||||
it('round-trips block-aligned data', async () => {
|
||||
await withDevice(async device => {
|
||||
const payload = Buffer.alloc(8 * BLOCK_SIZE)
|
||||
for (let i = 0; i < payload.length; i++) {
|
||||
payload[i] = bytePattern(i)
|
||||
}
|
||||
await device.write(16 * BLOCK_SIZE, payload)
|
||||
assert.deepEqual(await device.read(16 * BLOCK_SIZE, payload.length), payload)
|
||||
})
|
||||
})
|
||||
|
||||
it('round-trips data straddling block boundaries, at an unaligned offset', async () => {
|
||||
await withDevice(async device => {
|
||||
const payload = Buffer.from('a payload that is not a whole number of sectors long')
|
||||
const offset = 3 * BLOCK_SIZE + 17
|
||||
await device.write(offset, payload)
|
||||
assert.deepEqual(await device.read(offset, payload.length), payload)
|
||||
// the surrounding sector was read-modify-written by the kernel, not clobbered
|
||||
assert.deepEqual(await device.read(3 * BLOCK_SIZE, 17), Buffer.alloc(17))
|
||||
})
|
||||
})
|
||||
|
||||
it('survives a reopen, which is what makes it a cache', async () => {
|
||||
const payload = Buffer.alloc(BLOCK_SIZE, 0x5a)
|
||||
await withDevice(device => device.write(2 * BLOCK_SIZE, payload))
|
||||
await withDevice(async device => {
|
||||
await device.flush()
|
||||
assert.deepEqual(await device.read(2 * BLOCK_SIZE, BLOCK_SIZE), payload)
|
||||
})
|
||||
})
|
||||
|
||||
it('reads a large range in one call', async () => {
|
||||
await withDevice(async device => {
|
||||
const payload = Buffer.alloc(SIZE / 2)
|
||||
for (let i = 0; i < payload.length; i++) {
|
||||
payload[i] = bytePattern(i + 7)
|
||||
}
|
||||
await device.write(0, payload)
|
||||
assert.deepEqual(await device.read(0, payload.length), payload)
|
||||
})
|
||||
})
|
||||
|
||||
it('throws on a read running past the end, instead of zero-filling it', async () => {
|
||||
// the behaviour that sets the block device mode apart: a short read on a
|
||||
// device of a known size is an I/O error, never a sparse tail
|
||||
await withDevice(async device => {
|
||||
await assert.rejects(device.read(SIZE, BLOCK_SIZE), /ended after 0 bytes/)
|
||||
await assert.rejects(device.read(SIZE - BLOCK_SIZE, 2 * BLOCK_SIZE), /ended after 512 bytes/)
|
||||
})
|
||||
})
|
||||
|
||||
it('flushes without error', async () => {
|
||||
await withDevice(async device => {
|
||||
await device.write(0, Buffer.alloc(BLOCK_SIZE, 0xff))
|
||||
await device.flush()
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -16,11 +16,6 @@ export interface BlockDevice {
|
||||
getSize(): number
|
||||
/** Logical block size in bytes (typically 512). */
|
||||
getBlockSize(): number
|
||||
/**
|
||||
* When true, the LUN is reported as write-protected and WRITE commands are
|
||||
* refused up front (DATA PROTECT), without soliciting their data.
|
||||
*/
|
||||
readonly readOnly?: boolean
|
||||
/** Read exactly `length` bytes starting at byte `offset`. */
|
||||
read(offset: number, length: number): Promise<Buffer>
|
||||
/** Write `data` starting at byte `offset`. */
|
||||
@@ -34,60 +29,31 @@ export interface BlockDevice {
|
||||
const DEFAULT_BLOCK_SIZE = 512
|
||||
|
||||
export interface FileBlockDeviceOptions {
|
||||
/**
|
||||
* Path to the backing file. It must already exist and be sized to capacity —
|
||||
* or be a block device node, see {@link FileBlockDeviceOptions.size}.
|
||||
*/
|
||||
/** Path to the backing file. It must already exist and be sized to capacity. */
|
||||
readonly path: string
|
||||
/** Logical block size in bytes. Defaults to 512. */
|
||||
readonly blockSize?: number
|
||||
/**
|
||||
* Capacity in bytes, for a block device node (e.g. a VDI hot-plugged onto the
|
||||
* appliance's VM, `/dev/xvdX`), whose inode reports a size of 0.
|
||||
*
|
||||
* Setting it switches to the block device mode:
|
||||
* - `path` must be a block device: a typo naming the wrong node is the
|
||||
* difference between a cache and a destroyed disk;
|
||||
* - a short read is an I/O error, not a sparse tail: zero-filling it would hand
|
||||
* the initiator fabricated data.
|
||||
*/
|
||||
readonly size?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link BlockDevice} backed by a regular (sparse) file, or a block device
|
||||
* node, opened read/write.
|
||||
* A {@link BlockDevice} backed by a regular (sparse) file opened read/write.
|
||||
*
|
||||
* The file must already exist and be sized to the desired capacity (e.g. via
|
||||
* `fs.truncate`); this keeps the device a pure data store with no provisioning
|
||||
* policy of its own.
|
||||
*
|
||||
* I/O is buffered, never `O_DIRECT`, which would need buffers aligned to the
|
||||
* logical block size that Node cannot allocate.
|
||||
*/
|
||||
export class FileBlockDevice implements BlockDevice {
|
||||
readonly #path: string
|
||||
readonly #blockSize: number
|
||||
readonly #isBlockDevice: boolean
|
||||
#handle?: FileHandle
|
||||
#size = 0
|
||||
|
||||
constructor({ path, blockSize = DEFAULT_BLOCK_SIZE, size }: FileBlockDeviceOptions) {
|
||||
constructor({ path, blockSize = DEFAULT_BLOCK_SIZE }: FileBlockDeviceOptions) {
|
||||
if (!Number.isInteger(blockSize) || blockSize <= 0) {
|
||||
throw new Error(`blockSize must be a positive integer, got ${blockSize}`)
|
||||
}
|
||||
if (size !== undefined) {
|
||||
if (!Number.isInteger(size) || size <= 0) {
|
||||
throw new Error(`size must be a positive integer, got ${size}`)
|
||||
}
|
||||
if (size % blockSize !== 0) {
|
||||
throw new Error(`size (${size}) is not a multiple of block size (${blockSize})`)
|
||||
}
|
||||
this.#size = size
|
||||
}
|
||||
this.#path = path
|
||||
this.#blockSize = blockSize
|
||||
this.#isBlockDevice = size !== undefined
|
||||
}
|
||||
|
||||
/** Open the backing file and read its current size. Must be called before any I/O. */
|
||||
@@ -97,18 +63,12 @@ export class FileBlockDevice implements BlockDevice {
|
||||
}
|
||||
const handle = await open(this.#path, 'r+')
|
||||
try {
|
||||
const stats = await handle.stat()
|
||||
if (this.#isBlockDevice) {
|
||||
if (!stats.isBlockDevice()) {
|
||||
throw new Error(`${this.#path} is not a block device`)
|
||||
}
|
||||
} else {
|
||||
if (stats.size % this.#blockSize !== 0) {
|
||||
throw new Error(`backing file size (${stats.size}) is not a multiple of block size (${this.#blockSize})`)
|
||||
}
|
||||
this.#size = stats.size
|
||||
const { size } = await handle.stat()
|
||||
if (size % this.#blockSize !== 0) {
|
||||
throw new Error(`backing file size (${size}) is not a multiple of block size (${this.#blockSize})`)
|
||||
}
|
||||
this.#handle = handle
|
||||
this.#size = size
|
||||
} catch (error) {
|
||||
await handle.close()
|
||||
throw error
|
||||
@@ -139,10 +99,6 @@ export class FileBlockDevice implements BlockDevice {
|
||||
while (read < length) {
|
||||
const { bytesRead } = await handle.read(buffer, read, length - read, offset + read)
|
||||
if (bytesRead === 0) {
|
||||
if (this.#isBlockDevice) {
|
||||
// the device is exactly as long as the caller said: the I/O failed
|
||||
throw new Error(`read of ${length} bytes at ${offset} on ${this.#path} ended after ${read} bytes`)
|
||||
}
|
||||
// Reading past EOF (sparse tail): zero-fill the remainder.
|
||||
buffer.fill(0, read)
|
||||
break
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { FileBlockDevice } from './backend.mjs'
|
||||
|
||||
const BLOCK_SIZE = 512
|
||||
|
||||
/** A temporary directory removed whatever the test does. */
|
||||
const withTmpDir = async (fn: (dir: string) => Promise<void>): Promise<void> => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-backend-'))
|
||||
try {
|
||||
await fn(dir)
|
||||
} finally {
|
||||
await rm(dir, { force: true, recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** A sparse file of `size` bytes, as the class requires: existing and pre-sized. */
|
||||
const makeFile = async (dir: string, size: number, name = 'lun.img'): Promise<string> => {
|
||||
const path = join(dir, name)
|
||||
await writeFile(path, '')
|
||||
await truncate(path, size)
|
||||
return path
|
||||
}
|
||||
|
||||
describe('FileBlockDevice', () => {
|
||||
it('rejects a non-positive-integer block size', () => {
|
||||
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: 0 }), /positive integer/)
|
||||
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: -1 }), /positive integer/)
|
||||
assert.throws(() => new FileBlockDevice({ path: '/nope', blockSize: 1.5 }), /positive integer/)
|
||||
})
|
||||
|
||||
it('reads the capacity from the backing file', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, 4 * BLOCK_SIZE) })
|
||||
await device.open()
|
||||
try {
|
||||
assert.equal(device.getSize(), 4 * BLOCK_SIZE)
|
||||
assert.equal(device.getBlockSize(), BLOCK_SIZE)
|
||||
} finally {
|
||||
await device.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('refuses a file whose size is not a multiple of the block size', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE + 1) })
|
||||
await assert.rejects(device.open(), /not a multiple of block size/)
|
||||
})
|
||||
})
|
||||
|
||||
it('throws on I/O before open()', async () => {
|
||||
const device = new FileBlockDevice({ path: '/nope' })
|
||||
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
|
||||
await assert.rejects(device.write(0, Buffer.alloc(BLOCK_SIZE)), /must be called before I\/O/)
|
||||
await assert.rejects(device.flush(), /must be called before I\/O/)
|
||||
})
|
||||
|
||||
it('round-trips a write, at an offset and unaligned', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, 4 * BLOCK_SIZE) })
|
||||
await device.open()
|
||||
try {
|
||||
const payload = Buffer.from('hello, lun')
|
||||
await device.write(BLOCK_SIZE + 3, payload)
|
||||
assert.deepEqual(await device.read(BLOCK_SIZE + 3, payload.length), payload)
|
||||
|
||||
// the bytes around it are untouched
|
||||
assert.deepEqual(await device.read(BLOCK_SIZE, 3), Buffer.alloc(3))
|
||||
} finally {
|
||||
await device.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('zero-fills the sparse tail of a read past EOF', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
// a hole: nothing was ever written, so the whole file reads as zeroes
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, 2 * BLOCK_SIZE) })
|
||||
await device.open()
|
||||
try {
|
||||
assert.deepEqual(await device.read(0, 2 * BLOCK_SIZE), Buffer.alloc(2 * BLOCK_SIZE))
|
||||
|
||||
// past the end of the file, `fh.read` reports 0 bytes read: the remainder is
|
||||
// zero-filled rather than being reported as an error
|
||||
assert.deepEqual(await device.read(2 * BLOCK_SIZE, BLOCK_SIZE), Buffer.alloc(BLOCK_SIZE))
|
||||
} finally {
|
||||
await device.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('reads back a large buffer in full', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const size = 1024 * BLOCK_SIZE
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, size) })
|
||||
await device.open()
|
||||
try {
|
||||
const payload = Buffer.alloc(size)
|
||||
for (let i = 0; i < payload.length; i++) {
|
||||
payload[i] = (i * 31 + 11) & 0xff
|
||||
}
|
||||
await device.write(0, payload)
|
||||
assert.deepEqual(await device.read(0, size), payload)
|
||||
} finally {
|
||||
await device.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('flushes and closes, and tolerates being opened or closed twice', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE) })
|
||||
await device.open()
|
||||
await device.open() // no-op
|
||||
await device.write(0, Buffer.alloc(BLOCK_SIZE, 0xaa))
|
||||
await device.flush()
|
||||
await device.close()
|
||||
await device.close() // no-op
|
||||
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves no handle behind when open() fails', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const device = new FileBlockDevice({ path: await makeFile(dir, BLOCK_SIZE + 1) })
|
||||
await assert.rejects(device.open())
|
||||
// the handle opened before the size check was closed again
|
||||
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// A real device node cannot be created without root, so what is covered here is
|
||||
// everything around the I/O: the guards, the capacity, and the lifecycle. The
|
||||
// I/O itself is covered by `backend.integ.mts`, against a loop device.
|
||||
describe('FileBlockDevice with a size, over a block device', () => {
|
||||
it('rejects a non-positive-integer size', () => {
|
||||
for (const size of [0, -BLOCK_SIZE, 1.5]) {
|
||||
assert.throws(() => new FileBlockDevice({ path: '/dev/null', size }), /size must be a positive integer/)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects a size that is not a multiple of the block size', () => {
|
||||
assert.throws(
|
||||
() => new FileBlockDevice({ path: '/dev/null', size: BLOCK_SIZE + 1 }),
|
||||
/is not a multiple of block size/
|
||||
)
|
||||
})
|
||||
|
||||
it('reports the capacity it was given, never the inode', () => {
|
||||
// nothing may be read from the inode, which for a real device node reports 0
|
||||
const device = new FileBlockDevice({ path: '/dev/null', size: 4 * BLOCK_SIZE })
|
||||
assert.equal(device.getSize(), 4 * BLOCK_SIZE)
|
||||
assert.equal(device.getBlockSize(), BLOCK_SIZE)
|
||||
})
|
||||
|
||||
it('refuses to open a path that is not a block device', async () => {
|
||||
await withTmpDir(async dir => {
|
||||
const path = await makeFile(dir, 4 * BLOCK_SIZE)
|
||||
const device = new FileBlockDevice({ path, size: 4 * BLOCK_SIZE })
|
||||
// the guard against a misconfigured path naming, say, a real disk's
|
||||
// filesystem instead of the cache device
|
||||
await assert.rejects(device.open(), new RegExp(`${path} is not a block device`))
|
||||
// and the handle it opened to check was closed again
|
||||
await assert.rejects(device.read(0, BLOCK_SIZE), /must be called before I\/O/)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -24,13 +24,7 @@ import {
|
||||
TEXT_FLAG_FINAL,
|
||||
} from './constants.mjs'
|
||||
import { buildFixedSense, handleScsiCommand, type ScsiIdentity } from './scsi.mjs'
|
||||
import {
|
||||
buildSendTargetsResponse,
|
||||
LOGIN_MAX_DATA_SEGMENT_LENGTH,
|
||||
LoginNegotiator,
|
||||
OUR_MAX_RECV_DATA_SEGMENT_LENGTH,
|
||||
parseTextKeys,
|
||||
} from './login.mjs'
|
||||
import { buildSendTargetsResponse, LoginNegotiator, parseTextKeys } from './login.mjs'
|
||||
import { allocBhs, assemblePdu, type IncomingPdu, readPdu, writePdu } from './pdu.mjs'
|
||||
import {
|
||||
decodeCdb,
|
||||
@@ -87,16 +81,6 @@ export interface ConnectionDeps {
|
||||
allocateTsih(): number
|
||||
/** When set, require one-way CHAP: the target challenges and verifies the initiator. */
|
||||
readonly chap?: ChapCredentials
|
||||
/**
|
||||
* Time allowed to establish a Normal session, in ms (0 or unset disables):
|
||||
* past it, the connection is closed. Also bounds Discovery sessions, which
|
||||
* never become Normal ones.
|
||||
*/
|
||||
readonly loginTimeoutMs?: number
|
||||
/** Called once this connection has established a Normal session. */
|
||||
onSessionEstablished?(connection: Connection): void
|
||||
/** Max length of a single READ or WRITE, in bytes. */
|
||||
readonly maxTransferLength?: number
|
||||
}
|
||||
|
||||
type Phase = 'login' | 'fullFeature' | 'closed'
|
||||
@@ -139,8 +123,6 @@ export class Connection implements CommandContext {
|
||||
readonly #pendingWrites = new Map<number, PendingWrite>()
|
||||
#nextTargetTransferTag = 1
|
||||
|
||||
#loginTimer?: ReturnType<typeof setTimeout>
|
||||
|
||||
constructor(socket: Socket, deps: ConnectionDeps) {
|
||||
this.#socket = socket
|
||||
this.#deps = deps
|
||||
@@ -160,19 +142,9 @@ export class Connection implements CommandContext {
|
||||
|
||||
/** Read and service PDUs until the peer logs out or the connection drops. */
|
||||
async serve(): Promise<void> {
|
||||
const { loginTimeoutMs = 0 } = this.#deps
|
||||
if (loginTimeoutMs > 0) {
|
||||
this.#loginTimer = setTimeout(() => {
|
||||
log.warn('no session established in time, closing the connection', { remote: this.#socket.remoteAddress })
|
||||
this.destroy()
|
||||
}, loginTimeoutMs).unref()
|
||||
}
|
||||
try {
|
||||
for (;;) {
|
||||
const pdu = await readPdu(
|
||||
this.#socket,
|
||||
this.#phase === 'login' ? LOGIN_MAX_DATA_SEGMENT_LENGTH : OUR_MAX_RECV_DATA_SEGMENT_LENGTH
|
||||
)
|
||||
const pdu = await readPdu(this.#socket)
|
||||
if (pdu === null) {
|
||||
break // peer closed the connection
|
||||
}
|
||||
@@ -184,7 +156,6 @@ export class Connection implements CommandContext {
|
||||
} catch (error) {
|
||||
this.#fail(error)
|
||||
} finally {
|
||||
clearTimeout(this.#loginTimer)
|
||||
this.#phase = 'closed'
|
||||
this.#socket.destroy()
|
||||
}
|
||||
@@ -328,13 +299,6 @@ export class Connection implements CommandContext {
|
||||
bhs[37] = statusDetail
|
||||
await this.#send(assemblePdu(bhs, data))
|
||||
|
||||
// destroyed while the response was being written (login timeout, target
|
||||
// closing): the write still succeeds, but this connection must not go on
|
||||
// and replace the current session
|
||||
if (this.#phase === 'closed') {
|
||||
return true
|
||||
}
|
||||
|
||||
if (statusClass !== LoginStatusClass.SUCCESS) {
|
||||
// RFC 7143 §6.3: the failure response goes out first (above), then the
|
||||
// target closes the connection.
|
||||
@@ -352,10 +316,6 @@ export class Connection implements CommandContext {
|
||||
sessionType: this.#sessionType,
|
||||
target: this.#login.targetName,
|
||||
})
|
||||
if (this.#sessionType === 'Normal') {
|
||||
clearTimeout(this.#loginTimer)
|
||||
this.#deps.onSessionEstablished?.(this)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -429,13 +389,13 @@ export class Connection implements CommandContext {
|
||||
// Not awaited: the read loop must move on immediately rather than block on
|
||||
// this `lun.read()`. See the class doc for why no extra ordering is needed.
|
||||
void this.#readGate
|
||||
.run(() => handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity, this.#deps.maxTransferLength))
|
||||
.run(() => handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity))
|
||||
.catch(error => {
|
||||
this.#fail(error)
|
||||
})
|
||||
return
|
||||
}
|
||||
await handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity, this.#deps.maxTransferLength)
|
||||
await handleScsiCommand(cdb, pdu.itt, this, this.#deps.identity)
|
||||
}
|
||||
|
||||
async #sendReject(pdu: IncomingPdu, reason: number): Promise<void> {
|
||||
|
||||
@@ -166,7 +166,6 @@ export const SenseKey = {
|
||||
HARDWARE_ERROR: 0x04,
|
||||
ILLEGAL_REQUEST: 0x05,
|
||||
UNIT_ATTENTION: 0x06,
|
||||
DATA_PROTECT: 0x07,
|
||||
} as const
|
||||
|
||||
/** Additional Sense Code / Qualifier pairs we may emit. */
|
||||
@@ -175,7 +174,6 @@ export const Asc = {
|
||||
INVALID_COMMAND_OPERATION_CODE: { asc: 0x20, ascq: 0x00 },
|
||||
INVALID_FIELD_IN_CDB: { asc: 0x24, ascq: 0x00 },
|
||||
LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE: { asc: 0x21, ascq: 0x00 },
|
||||
WRITE_PROTECTED: { asc: 0x27, ascq: 0x00 },
|
||||
} as const
|
||||
|
||||
/** Reject reason codes (Reject PDU byte 2). */
|
||||
|
||||
@@ -3,7 +3,7 @@ import { createLogger, type Logger } from '@xen-orchestra/log'
|
||||
|
||||
import type { BlockDevice } from './backend.mjs'
|
||||
import { Connection, type ConnectionDeps } from './connection.mjs'
|
||||
import { DEFAULT_MAX_TRANSFER_LENGTH, type ScsiIdentity } from './scsi.mjs'
|
||||
import type { ScsiIdentity } from './scsi.mjs'
|
||||
import type { ChapCredentials } from './types.mjs'
|
||||
|
||||
export { FileBlockDevice } from './backend.mjs'
|
||||
@@ -25,8 +25,6 @@ const DEFAULT_PORT = 3260
|
||||
const DEFAULT_WRITE_TIMEOUT_MS = 30_000
|
||||
const DEFAULT_CMD_WINDOW = 64
|
||||
const DEFAULT_READ_CONCURRENCY = 16
|
||||
const DEFAULT_LOGIN_TIMEOUT_MS = 15_000
|
||||
const DEFAULT_MAX_PENDING_CONNECTIONS = 8
|
||||
|
||||
const DEFAULT_IDENTITY: ScsiIdentity = {
|
||||
vendor: 'VATES',
|
||||
@@ -63,62 +61,18 @@ export interface IscsiTargetOptions {
|
||||
* open-iscsi `node.session.auth` username/password). Omit for no authentication.
|
||||
*/
|
||||
readonly chap?: ChapCredentials
|
||||
/**
|
||||
* Time a new connection has to establish a Normal session, in ms (0
|
||||
* disables). Also bounds Discovery sessions. Defaults to 15000.
|
||||
*/
|
||||
readonly loginTimeoutMs?: number
|
||||
/**
|
||||
* Max number of connections that have not established a Normal session yet;
|
||||
* past it, new ones are refused. Defaults to 8.
|
||||
*/
|
||||
readonly maxPendingConnections?: number
|
||||
/**
|
||||
* Source addresses the initiators may connect from. When set, connections
|
||||
* from any other address are closed as soon as they are accepted, before any
|
||||
* login: only these initiators can take a pending connection slot.
|
||||
* IPv4-mapped IPv6 addresses (`::ffff:a.b.c.d`) match their IPv4 form.
|
||||
*/
|
||||
readonly allowedInitiatorAddresses?: Iterable<string>
|
||||
/**
|
||||
* Called with the source address of each connection closed because it is
|
||||
* not in `allowedInitiatorAddresses`, e.g. to tell why an initiator behind a
|
||||
* NAT cannot connect.
|
||||
*/
|
||||
readonly onConnectionRefused?: (address: string) => void
|
||||
/**
|
||||
* Max length of a single READ or WRITE, in bytes, advertised in the block
|
||||
* limits VPD page; longer commands are refused. At least one logical block.
|
||||
* Defaults to 8 MiB.
|
||||
*/
|
||||
readonly maxTransferLength?: number
|
||||
}
|
||||
|
||||
const IPV4_MAPPED_PREFIX = '::ffff:'
|
||||
|
||||
function normalizeAddress(address: string): string {
|
||||
return address.startsWith(IPV4_MAPPED_PREFIX) && address.includes('.')
|
||||
? address.slice(IPV4_MAPPED_PREFIX.length)
|
||||
: address
|
||||
}
|
||||
|
||||
/**
|
||||
* A minimal iSCSI target exposing exactly one read/write LUN.
|
||||
*
|
||||
* Single initiator, single connection (`MaxConnections=1`),
|
||||
* `ErrorRecoveryLevel=0`, no digests.
|
||||
*
|
||||
* A new connection replaces the current one rather than being refused (see
|
||||
* `#onSessionEstablished`): there is no way to tell a stale, abandoned
|
||||
* connection (initiator gave up without closing the socket) from a healthy
|
||||
* one, so refusing would let one stuck connection wedge the target forever.
|
||||
* The replacement only happens once the new connection has established a
|
||||
* Normal session, CHAP included when configured, so that a port scan, a failed
|
||||
* login or a Discovery session cannot interrupt the current one. Until then,
|
||||
* connections are bounded in time (`loginTimeoutMs`) and in number
|
||||
* (`maxPendingConnections`); with `allowedInitiatorAddresses`, only the expected
|
||||
* initiators can take those slots. Not RFC 7143 session reinstatement (no ISID
|
||||
* matching) — safe here since the target is ephemeral, CHAP-guarded, and
|
||||
* `ErrorRecoveryLevel=0`, no digests. A new connection always replaces the
|
||||
* current one rather than being refused (see `#onConnection`): there is no
|
||||
* way to tell a stale, abandoned connection (initiator gave up without
|
||||
* closing the socket) from a healthy one, so refusing would let one stuck
|
||||
* connection wedge the target forever. Not RFC 7143 session reinstatement (no
|
||||
* ISID matching) — safe here since the target is ephemeral, CHAP-guarded, and
|
||||
* single-consumer.
|
||||
*/
|
||||
export class IscsiTarget {
|
||||
@@ -131,17 +85,9 @@ export class IscsiTarget {
|
||||
readonly #cmdWindow: number
|
||||
readonly #readConcurrency: number
|
||||
readonly #chap?: ChapCredentials
|
||||
readonly #loginTimeoutMs: number
|
||||
readonly #maxPendingConnections: number
|
||||
readonly #allowedInitiatorAddresses?: ReadonlySet<string>
|
||||
readonly #onConnectionRefused?: (address: string) => void
|
||||
readonly #maxTransferLength: number
|
||||
|
||||
#server?: Server
|
||||
// the connection with an established Normal session, serving the LUN
|
||||
#connection?: Connection
|
||||
// every open connection, so that close() can drop them all
|
||||
readonly #connections = new Set<Connection>()
|
||||
#tsih = 0
|
||||
|
||||
constructor(options: IscsiTargetOptions) {
|
||||
@@ -153,16 +99,6 @@ export class IscsiTarget {
|
||||
this.#cmdWindow = options.cmdWindow ?? DEFAULT_CMD_WINDOW
|
||||
this.#readConcurrency = options.readConcurrency ?? DEFAULT_READ_CONCURRENCY
|
||||
this.#chap = options.chap
|
||||
this.#loginTimeoutMs = options.loginTimeoutMs ?? DEFAULT_LOGIN_TIMEOUT_MS
|
||||
this.#maxPendingConnections = options.maxPendingConnections ?? DEFAULT_MAX_PENDING_CONNECTIONS
|
||||
if (options.allowedInitiatorAddresses !== undefined) {
|
||||
this.#allowedInitiatorAddresses = new Set(Array.from(options.allowedInitiatorAddresses, normalizeAddress))
|
||||
}
|
||||
this.#onConnectionRefused = options.onConnectionRefused
|
||||
this.#maxTransferLength = options.maxTransferLength ?? DEFAULT_MAX_TRANSFER_LENGTH
|
||||
if (this.#maxTransferLength < options.lun.getBlockSize()) {
|
||||
throw new RangeError('maxTransferLength must be at least one logical block')
|
||||
}
|
||||
this.#identity = {
|
||||
...DEFAULT_IDENTITY,
|
||||
// Default the serial to the IQN so the LUN has a stable, unique identity.
|
||||
@@ -186,44 +122,28 @@ export class IscsiTarget {
|
||||
readConcurrency: this.#readConcurrency,
|
||||
allocateTsih: () => this.#allocateTsih(),
|
||||
chap: this.#chap,
|
||||
loginTimeoutMs: this.#loginTimeoutMs,
|
||||
onSessionEstablished: connection => this.#onSessionEstablished(connection),
|
||||
maxTransferLength: this.#maxTransferLength,
|
||||
}
|
||||
}
|
||||
|
||||
#onConnection(socket: Socket): void {
|
||||
const allowed = this.#allowedInitiatorAddresses
|
||||
const remote = normalizeAddress(socket.remoteAddress ?? '')
|
||||
if (allowed !== undefined && !allowed.has(remote)) {
|
||||
log.warn('connection from an address that is not allowed, closing it', { remote })
|
||||
socket.destroy()
|
||||
this.#onConnectionRefused?.(remote)
|
||||
return
|
||||
const existing = this.#connection
|
||||
if (existing !== undefined) {
|
||||
// Replaces a possibly-stale connection rather than refusing; see the class doc.
|
||||
log.warn('replacing existing connection with a new one', { remote: socket.remoteAddress })
|
||||
existing.destroy()
|
||||
}
|
||||
socket.setNoDelay(true)
|
||||
const connection = new Connection(socket, this.#deps())
|
||||
this.#connections.add(connection)
|
||||
this.#connection = connection
|
||||
// Long-lived per-connection task; serve() handles its own errors and never
|
||||
// rejects, so detaching it here is safe.
|
||||
void connection.serve().finally(() => {
|
||||
this.#connections.delete(connection)
|
||||
if (this.#connection === connection) {
|
||||
this.#connection = undefined
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#onSessionEstablished(connection: Connection): void {
|
||||
const existing = this.#connection
|
||||
this.#connection = connection
|
||||
if (existing !== undefined) {
|
||||
// Replaces a possibly-stale connection rather than refusing; see the class doc.
|
||||
log.warn('replacing existing connection with a newly logged in one')
|
||||
existing.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the LUN (if needed) and start accepting connections. */
|
||||
async listen(): Promise<void> {
|
||||
if (this.#server !== undefined) {
|
||||
@@ -232,11 +152,6 @@ export class IscsiTarget {
|
||||
await this.#lun.open?.()
|
||||
|
||||
const server = createServer(socket => this.#onConnection(socket))
|
||||
// + 1: the connection serving the LUN does not take a pending slot
|
||||
server.maxConnections = this.#maxPendingConnections + 1
|
||||
server.on('drop', data => {
|
||||
log.warn('too many connections without a session, refusing a new one', { remote: data?.remoteAddress })
|
||||
})
|
||||
this.#server = server
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -260,20 +175,17 @@ export class IscsiTarget {
|
||||
return address !== null && typeof address === 'object' ? address : undefined
|
||||
}
|
||||
|
||||
/** Stop accepting connections, drop every connection, and close the LUN. */
|
||||
/** Stop accepting connections, drop the active connection, and close the LUN. */
|
||||
async close(): Promise<void> {
|
||||
const server = this.#server
|
||||
const connections = [...this.#connections]
|
||||
const connection = this.#connection
|
||||
this.#server = undefined
|
||||
this.#connection = undefined
|
||||
this.#connections.clear()
|
||||
if (server !== undefined) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close(error => (error ? reject(error) : resolve()))
|
||||
// Drop the live connections so server.close() can complete.
|
||||
for (const connection of connections) {
|
||||
connection.destroy()
|
||||
}
|
||||
// Drop the live connection so server.close() can complete.
|
||||
connection?.destroy()
|
||||
})
|
||||
}
|
||||
await this.#lun.close()
|
||||
|
||||
@@ -35,12 +35,9 @@ const log: Logger = createLogger('vates:iscsi:login')
|
||||
// Operational values the target imposes. The target drives negotiation, so these
|
||||
// are pinned to a single code path (see plan): all writes are R2T-solicited
|
||||
// (InitialR2T=Yes, ImmediateData=No), no digests, one connection.
|
||||
export const OUR_MAX_RECV_DATA_SEGMENT_LENGTH = 262144
|
||||
const OUR_MAX_RECV_DATA_SEGMENT_LENGTH = 262144
|
||||
const OUR_MAX_BURST_LENGTH = 262144
|
||||
const DEFAULT_INITIATOR_MAX_RECV_DATA_SEGMENT_LENGTH = 8192 // RFC 7143 default
|
||||
// a declared MaxRecvDataSegmentLength only applies once logged in: until then,
|
||||
// both sides keep to the 8192-byte default (RFC 7143 §13.12)
|
||||
export const LOGIN_MAX_DATA_SEGMENT_LENGTH = 8192
|
||||
|
||||
/** Parse a NUL-delimited iSCSI text key=value data segment. */
|
||||
export function parseTextKeys(data: Buffer): Map<string, string> {
|
||||
|
||||
@@ -4,7 +4,6 @@ import { connect, type Socket } from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { once } from 'node:events'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
|
||||
import {
|
||||
@@ -643,200 +642,6 @@ describe('connection replacement', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// A connection only takes over once it has itself logged in: until then, it
|
||||
// must not disturb the session in use, whatever it does or fails to do.
|
||||
describe('connection admission', () => {
|
||||
const CHAP = { user: 'alice', secret: 's3cr3t-s3cr3t' }
|
||||
|
||||
let dir: string
|
||||
let backingPath: string
|
||||
|
||||
before(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'vates-iscsi-admission-'))
|
||||
backingPath = join(dir, 'lun.img')
|
||||
await writeFile(backingPath, Buffer.alloc(LUN_SIZE, 0x5a))
|
||||
})
|
||||
|
||||
after(async () => {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function withTarget(
|
||||
options: {
|
||||
loginTimeoutMs?: number
|
||||
maxPendingConnections?: number
|
||||
allowedInitiatorAddresses?: string[]
|
||||
onConnectionRefused?: (address: string) => void
|
||||
},
|
||||
fn: (port: number) => Promise<void>
|
||||
): Promise<void> {
|
||||
const target = new IscsiTarget({
|
||||
iqn: IQN,
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
lun: new FileBlockDevice({ path: backingPath, blockSize: BLOCK_SIZE }),
|
||||
chap: CHAP,
|
||||
...options,
|
||||
})
|
||||
await target.listen()
|
||||
try {
|
||||
const address = target.address()
|
||||
assert.ok(address !== undefined)
|
||||
await fn(address.port)
|
||||
} finally {
|
||||
await target.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function rawConnect(port: number): Promise<Socket> {
|
||||
const socket = connect(port, '127.0.0.1')
|
||||
await once(socket, 'connect')
|
||||
return socket
|
||||
}
|
||||
|
||||
async function assertStillServing(initiator: IscsiInitiator): Promise<void> {
|
||||
// leave the target time to act on whatever just happened
|
||||
await delay(100)
|
||||
assert.deepEqual(await initiator.read(0, BLOCK_SIZE), Buffer.alloc(BLOCK_SIZE, 0x5a))
|
||||
}
|
||||
|
||||
it('a connection that sends nothing does not interrupt the session', async () => {
|
||||
await withTarget({}, async port => {
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
const probe = await rawConnect(port)
|
||||
try {
|
||||
await assertStillServing(initiator)
|
||||
} finally {
|
||||
probe.destroy()
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('a failed CHAP login does not interrupt the session', async () => {
|
||||
await withTarget({}, async port => {
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
try {
|
||||
const intruder = new IscsiInitiator({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
targetIqn: IQN,
|
||||
chap: { user: CHAP.user, secret: 'wrong-wrong-wrong' },
|
||||
})
|
||||
await assert.rejects(intruder.connect(), /login rejected/)
|
||||
await assertStillServing(initiator)
|
||||
} finally {
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('a Discovery session does not interrupt the session', async () => {
|
||||
await withTarget({}, async port => {
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
const socket = await rawConnect(port)
|
||||
try {
|
||||
const discovery = new MiniInitiator(socket)
|
||||
await discovery.login('Discovery')
|
||||
assert.equal((await discovery.sendTargets()).get('TargetName'), IQN)
|
||||
await assertStillServing(initiator)
|
||||
await discovery.logout()
|
||||
} finally {
|
||||
socket.destroy()
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('an authenticated login replaces the session', async () => {
|
||||
await withTarget({}, async port => {
|
||||
const stale = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await stale.connect()
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
try {
|
||||
await assert.rejects(stale.read(0, BLOCK_SIZE), /connection closed/)
|
||||
await assertStillServing(initiator)
|
||||
} finally {
|
||||
await stale.close().catch(() => {})
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('closes a connection that does not log in in time', async () => {
|
||||
await withTarget({ loginTimeoutMs: 100 }, async port => {
|
||||
const probe = await rawConnect(port)
|
||||
const closed = once(probe, 'close')
|
||||
try {
|
||||
await Promise.race([closed, delay(5000).then(() => assert.fail('the connection should have been closed'))])
|
||||
} finally {
|
||||
probe.destroy()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('closes connections from an address that is not allowed, and reports it', async () => {
|
||||
const refused: string[] = []
|
||||
await withTarget(
|
||||
{ allowedInitiatorAddresses: ['192.0.2.1'], onConnectionRefused: address => refused.push(address) },
|
||||
async port => {
|
||||
const probe = await rawConnect(port)
|
||||
try {
|
||||
await Promise.race([
|
||||
once(probe, 'close'),
|
||||
delay(5000).then(() => assert.fail('the connection should have been closed')),
|
||||
])
|
||||
} finally {
|
||||
probe.destroy()
|
||||
}
|
||||
}
|
||||
)
|
||||
assert.deepEqual(refused, ['127.0.0.1'])
|
||||
})
|
||||
|
||||
it('serves an allowed address, also in its IPv4-mapped IPv6 form', async () => {
|
||||
for (const allowed of ['127.0.0.1', '::ffff:127.0.0.1']) {
|
||||
await withTarget({ allowedInitiatorAddresses: [allowed] }, async port => {
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
try {
|
||||
await assertStillServing(initiator)
|
||||
} finally {
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
it('refuses connections without a session past the limit', async () => {
|
||||
await withTarget({ maxPendingConnections: 2 }, async port => {
|
||||
const initiator = new IscsiInitiator({ host: '127.0.0.1', port, targetIqn: IQN, chap: CHAP })
|
||||
await initiator.connect()
|
||||
const probes = [await rawConnect(port), await rawConnect(port)]
|
||||
try {
|
||||
const extra = await rawConnect(port)
|
||||
probes.push(extra)
|
||||
await Promise.race([
|
||||
once(extra, 'close'),
|
||||
delay(5000).then(() => assert.fail('the connection should have been refused')),
|
||||
])
|
||||
assert.equal(probes[0].destroyed, false)
|
||||
assert.equal(probes[1].destroyed, false)
|
||||
await assertStillServing(initiator)
|
||||
} finally {
|
||||
for (const probe of probes) {
|
||||
probe.destroy()
|
||||
}
|
||||
await initiator.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// One loopback that exercises BOTH one-way CHAP roles at once: our target is the
|
||||
// authenticator (challenges + verifies) and our initiator is the responder
|
||||
// (answers the challenge). This is the same code path each production role uses
|
||||
|
||||
@@ -86,12 +86,8 @@ export class IncomingPdu {
|
||||
* No per-read timeout, intentionally — a target may sit idle between commands
|
||||
* indefinitely. Liveness is a transport concern (socket timeout / NOP
|
||||
* keepalives), handled elsewhere.
|
||||
*
|
||||
* A PDU announcing a data segment longer than `maxDataSegmentLength` is
|
||||
* rejected before its payload is read, so a peer cannot make us buffer up to
|
||||
* the 16 MiB the 24-bit length field allows.
|
||||
*/
|
||||
export async function readPdu(stream: Readable, maxDataSegmentLength = Infinity): Promise<IncomingPdu | null> {
|
||||
export async function readPdu(stream: Readable): Promise<IncomingPdu | null> {
|
||||
const bhs = await readChunk(stream, BHS_LENGTH)
|
||||
if (bhs === null) {
|
||||
return null // clean EOF between PDUs
|
||||
@@ -102,9 +98,6 @@ export async function readPdu(stream: Readable, maxDataSegmentLength = Infinity)
|
||||
|
||||
const totalAhsLength = bhs.readUInt8(4)
|
||||
const dataSegmentLength = bhs.readUIntBE(5, 3)
|
||||
if (dataSegmentLength > maxDataSegmentLength) {
|
||||
throw new Error(`iSCSI data segment too long (${dataSegmentLength} > ${maxDataSegmentLength} bytes)`)
|
||||
}
|
||||
const ahsBytes = totalAhsLength * 4
|
||||
const dataBytesPadded = roundUp4(dataSegmentLength)
|
||||
|
||||
|
||||
@@ -56,24 +56,6 @@ describe('assemblePdu / readPdu round-trip', () => {
|
||||
})
|
||||
|
||||
describe('readPdu framing', () => {
|
||||
it('rejects a data segment longer than the limit before reading it', async () => {
|
||||
const bhs = allocBhs(InitiatorOpcode.SCSI_COMMAND)
|
||||
bhs.writeUIntBE(16 * 1024 * 1024 - 1, 5, 3) // announced data segment length
|
||||
const stream = new PassThrough()
|
||||
stream.write(bhs) // the announced payload is never sent
|
||||
await assert.rejects(readPdu(stream, 8192), /data segment too long/)
|
||||
})
|
||||
|
||||
it('accepts a data segment of exactly the limit', async () => {
|
||||
const data = Buffer.alloc(8192, 0x5a)
|
||||
const pdu = await (async () => {
|
||||
const stream = new PassThrough()
|
||||
stream.end(buildScsiCommandPdu(1, 1, Buffer.alloc(16), data))
|
||||
return readPdu(stream, 8192)
|
||||
})()
|
||||
assert.deepEqual(pdu?.data, data)
|
||||
})
|
||||
|
||||
it('reassembles a PDU split across multiple TCP chunks', async () => {
|
||||
const data = Buffer.from('hello world!!') // 13 bytes -> padded to 16
|
||||
const wire = buildScsiCommandPdu(7, 7, Buffer.alloc(16), data)
|
||||
|
||||
@@ -40,20 +40,13 @@ export function buildStandardInquiry(identity: ScsiIdentity): Buffer {
|
||||
return buffer
|
||||
}
|
||||
|
||||
const SUPPORTED_VPD_PAGES = [0x00, 0x80, 0x83, 0xb0]
|
||||
|
||||
/** Default max length of a single READ or WRITE, in bytes. */
|
||||
export const DEFAULT_MAX_TRANSFER_LENGTH = 8 * 1024 * 1024
|
||||
const SUPPORTED_VPD_PAGES = [0x00, 0x80, 0x83]
|
||||
|
||||
/**
|
||||
* Build an INQUIRY EVPD page, or `undefined` for an unsupported page code (the
|
||||
* caller then returns CHECK CONDITION / INVALID FIELD IN CDB).
|
||||
*/
|
||||
export function buildVpd(
|
||||
pageCode: number,
|
||||
identity: ScsiIdentity,
|
||||
maxTransferBlocks = DEFAULT_MAX_TRANSFER_LENGTH / 512
|
||||
): Buffer | undefined {
|
||||
export function buildVpd(pageCode: number, identity: ScsiIdentity): Buffer | undefined {
|
||||
switch (pageCode) {
|
||||
case 0x00: {
|
||||
// Supported VPD pages.
|
||||
@@ -90,15 +83,6 @@ export function buildVpd(
|
||||
descriptor.copy(buffer, 4)
|
||||
return buffer
|
||||
}
|
||||
case 0xb0: {
|
||||
// Block limits: initiators split their I/O to the max transfer length
|
||||
// instead of seeing it refused.
|
||||
const buffer = Buffer.alloc(64)
|
||||
buffer[1] = 0xb0
|
||||
buffer.writeUInt16BE(0x3c, 2)
|
||||
buffer.writeUInt32BE(maxTransferBlocks, 8) // maximum transfer length
|
||||
return buffer
|
||||
}
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
@@ -141,21 +125,15 @@ export function buildFixedSense({ key, asc, ascq }: SenseInfo): Buffer {
|
||||
return buffer
|
||||
}
|
||||
|
||||
// WP bit of the device-specific parameter in the mode parameter header
|
||||
const MODE_SENSE_WRITE_PROTECT = 0x80
|
||||
|
||||
/** Minimal MODE SENSE parameter data: a header with no block descriptor or pages. */
|
||||
export function buildModeSense(long: boolean, writeProtected = false): Buffer {
|
||||
const deviceSpecific = writeProtected ? MODE_SENSE_WRITE_PROTECT : 0
|
||||
export function buildModeSense(long: boolean): Buffer {
|
||||
if (long) {
|
||||
const buffer = Buffer.alloc(8)
|
||||
buffer.writeUInt16BE(6, 0) // mode data length (8 - 2)
|
||||
buffer[3] = deviceSpecific
|
||||
return buffer
|
||||
}
|
||||
const buffer = Buffer.alloc(4)
|
||||
buffer[0] = 3 // mode data length (4 - 1)
|
||||
buffer[2] = deviceSpecific
|
||||
return buffer
|
||||
}
|
||||
|
||||
@@ -172,7 +150,6 @@ const ILLEGAL_REQUEST_LBA_OUT_OF_RANGE: SenseInfo = {
|
||||
...Asc.LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE,
|
||||
}
|
||||
const MEDIUM_ERROR_NO_ADDITIONAL_SENSE: SenseInfo = { key: SenseKey.MEDIUM_ERROR, ...Asc.NO_ADDITIONAL_SENSE }
|
||||
const DATA_PROTECT_WRITE_PROTECTED: SenseInfo = { key: SenseKey.DATA_PROTECT, ...Asc.WRITE_PROTECTED }
|
||||
|
||||
function checkCondition(ctx: CommandContext, itt: number, sense: SenseInfo): Promise<void> {
|
||||
return ctx.sendScsiResponse(itt, ScsiStatus.CHECK_CONDITION, { sense: buildFixedSense(sense) })
|
||||
@@ -184,22 +161,16 @@ function checkCondition(ctx: CommandContext, itt: number, sense: SenseInfo): Pro
|
||||
* `cdb` is the 16-byte CDB from the SCSI Command PDU and `itt` its Initiator
|
||||
* Task Tag. Read commands stream Data-In; WRITE solicits Data-Out via R2T;
|
||||
* everything else returns status (with sense on error).
|
||||
*
|
||||
* A READ or WRITE longer than `maxTransferLength` bytes, advertised in the
|
||||
* block limits VPD page, is refused with INVALID FIELD IN CDB: it also bounds
|
||||
* the buffer a WRITE allocates.
|
||||
*/
|
||||
export async function handleScsiCommand(
|
||||
cdb: Buffer,
|
||||
itt: number,
|
||||
ctx: CommandContext,
|
||||
identity: ScsiIdentity,
|
||||
maxTransferLength = DEFAULT_MAX_TRANSFER_LENGTH
|
||||
identity: ScsiIdentity
|
||||
): Promise<void> {
|
||||
const { lun } = ctx
|
||||
const blockSize = lun.getBlockSize()
|
||||
const blockCount = lun.getSize() / blockSize
|
||||
const maxTransferBlocks = Math.floor(maxTransferLength / blockSize)
|
||||
const request = decodeCdb(cdb)
|
||||
log.debug('scsi command', { itt, ...request })
|
||||
|
||||
@@ -212,7 +183,7 @@ export async function handleScsiCommand(
|
||||
}
|
||||
return ctx.sendReadData(itt, buildStandardInquiry(identity), request.allocationLength)
|
||||
}
|
||||
const page = buildVpd(request.pageCode, identity, maxTransferBlocks)
|
||||
const page = buildVpd(request.pageCode, identity)
|
||||
if (page === undefined) {
|
||||
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
|
||||
}
|
||||
@@ -232,9 +203,6 @@ export async function handleScsiCommand(
|
||||
if (request.lba + request.blocks > blockCount) {
|
||||
return checkCondition(ctx, itt, ILLEGAL_REQUEST_LBA_OUT_OF_RANGE)
|
||||
}
|
||||
if (request.blocks > maxTransferBlocks) {
|
||||
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
|
||||
}
|
||||
const byteLength = request.blocks * blockSize
|
||||
if (byteLength === 0) {
|
||||
return ctx.sendReadData(itt, Buffer.alloc(0), 0)
|
||||
@@ -253,15 +221,9 @@ export async function handleScsiCommand(
|
||||
}
|
||||
|
||||
case 'write': {
|
||||
if (lun.readOnly === true) {
|
||||
return checkCondition(ctx, itt, DATA_PROTECT_WRITE_PROTECTED)
|
||||
}
|
||||
if (request.lba + request.blocks > blockCount) {
|
||||
return checkCondition(ctx, itt, ILLEGAL_REQUEST_LBA_OUT_OF_RANGE)
|
||||
}
|
||||
if (request.blocks > maxTransferBlocks) {
|
||||
return checkCondition(ctx, itt, ILLEGAL_REQUEST_INVALID_FIELD)
|
||||
}
|
||||
const byteLength = request.blocks * blockSize
|
||||
if (byteLength === 0) {
|
||||
return ctx.sendScsiResponse(itt, ScsiStatus.GOOD)
|
||||
@@ -281,7 +243,7 @@ export async function handleScsiCommand(
|
||||
}
|
||||
|
||||
case 'modeSense':
|
||||
return ctx.sendReadData(itt, buildModeSense(request.long, lun.readOnly === true), request.allocationLength)
|
||||
return ctx.sendReadData(itt, buildModeSense(request.long), request.allocationLength)
|
||||
|
||||
case 'syncCache':
|
||||
await lun.flush()
|
||||
|
||||
@@ -145,17 +145,9 @@ describe('response encoders', () => {
|
||||
it('advertises supported VPD pages and rejects unknown ones', () => {
|
||||
const page00 = buildVpd(0x00, IDENTITY)
|
||||
assert.ok(page00 !== undefined)
|
||||
assert.deepEqual([...page00.subarray(4)], [0x00, 0x80, 0x83, 0xb0])
|
||||
assert.deepEqual([...page00.subarray(4)], [0x00, 0x80, 0x83])
|
||||
assert.equal(buildVpd(0x55, IDENTITY), undefined)
|
||||
})
|
||||
|
||||
it('advertises the max transfer length in the block limits page', () => {
|
||||
const page = buildVpd(0xb0, IDENTITY, 2048)
|
||||
assert.ok(page !== undefined)
|
||||
assert.equal(page[1], 0xb0)
|
||||
assert.equal(page.readUInt16BE(2), 0x3c)
|
||||
assert.equal(page.readUInt32BE(8), 2048)
|
||||
})
|
||||
})
|
||||
|
||||
describe('handleScsiCommand', () => {
|
||||
@@ -242,61 +234,4 @@ describe('handleScsiCommand', () => {
|
||||
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
|
||||
assert.equal(ctx.responses[0].options?.sense?.[12], 0x20) // INVALID COMMAND OPERATION CODE
|
||||
})
|
||||
|
||||
for (const [name, opcode] of [
|
||||
['READ(10)', 0x28],
|
||||
['WRITE(10)', 0x2a],
|
||||
] as const) {
|
||||
it(`refuses a ${name} longer than the max transfer length, without reading or soliciting data`, async () => {
|
||||
let reads = 0
|
||||
const lun = new MemoryBlockDevice(8192)
|
||||
const read = lun.read.bind(lun)
|
||||
lun.read = (offset, length) => {
|
||||
reads++
|
||||
return read(offset, length)
|
||||
}
|
||||
const ctx = new FakeContext(lun)
|
||||
const buffer = Buffer.alloc(16)
|
||||
buffer[0] = opcode
|
||||
buffer.writeUInt16BE(3, 7) // 3 blocks, the limit is 2
|
||||
await handleScsiCommand(buffer, 1, ctx, IDENTITY, 1024)
|
||||
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
|
||||
assert.equal(ctx.responses[0].options?.sense?.[2], SenseKey.ILLEGAL_REQUEST)
|
||||
assert.equal(ctx.responses[0].options?.sense?.[12], 0x24) // INVALID FIELD IN CDB
|
||||
assert.equal(reads, 0)
|
||||
assert.equal(ctx.writes.length, 0)
|
||||
})
|
||||
}
|
||||
|
||||
describe('read-only LUN', () => {
|
||||
class ReadOnlyBlockDevice extends MemoryBlockDevice {
|
||||
readonly readOnly = true
|
||||
}
|
||||
|
||||
it('refuses WRITE(10) with DATA PROTECT, without soliciting its data', async () => {
|
||||
const ctx = new FakeContext(new ReadOnlyBlockDevice(4096))
|
||||
const buffer = Buffer.alloc(16)
|
||||
buffer[0] = 0x2a
|
||||
buffer.writeUInt16BE(1, 7)
|
||||
await handleScsiCommand(buffer, 1, ctx, IDENTITY)
|
||||
assert.equal(ctx.writes.length, 0)
|
||||
assert.equal(ctx.responses[0].status, ScsiStatus.CHECK_CONDITION)
|
||||
assert.equal(ctx.responses[0].options?.sense?.[2], SenseKey.DATA_PROTECT)
|
||||
assert.equal(ctx.responses[0].options?.sense?.[12], 0x27) // WRITE PROTECTED
|
||||
})
|
||||
|
||||
it('reports write protection in MODE SENSE(6) and MODE SENSE(10)', async () => {
|
||||
const ctx = new FakeContext(new ReadOnlyBlockDevice(4096))
|
||||
await handleScsiCommand(cdb(0x1a, 0x00, 0x3f, 0x00, 255), 1, ctx, IDENTITY)
|
||||
await handleScsiCommand(cdb(0x5a, 0x00, 0x3f, 0, 0, 0, 0, 0x01, 0x00), 2, ctx, IDENTITY)
|
||||
assert.equal(ctx.reads[0].data[2] & 0x80, 0x80)
|
||||
assert.equal(ctx.reads[1].data[3] & 0x80, 0x80)
|
||||
})
|
||||
|
||||
it('does not report a writable LUN as write-protected', async () => {
|
||||
const ctx = new FakeContext(new MemoryBlockDevice(4096))
|
||||
await handleScsiCommand(cdb(0x1a, 0x00, 0x3f, 0x00, 255), 1, ctx, IDENTITY)
|
||||
assert.equal(ctx.reads[0].data[2] & 0x80, 0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -56,21 +56,17 @@ function parseAttachedDisk(sessionOutput: string): string | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* Neither the device node nor a nonzero `/sys/class/block/<name>/size` means
|
||||
* the disk can be opened: the sd driver sets the size before
|
||||
* `device_add_disk()`, which creates the sysfs entry, then the `/dev` node, then
|
||||
* registers the block device: until then `open()` fails with ENOENT or ENXIO.
|
||||
* Only an `open()` that succeeds means ready.
|
||||
* `stat()`ing the device node only proves udev created it, not that the SCSI
|
||||
* layer has finished attaching it — `open()` right after can still fail with
|
||||
* ENXIO. `/sys/class/block/<name>/size` reporting nonzero is what actually
|
||||
* means ready, the same check `_cache.mjs`'s `openLocalDevice` uses for the
|
||||
* same reason on XAPI-plugged devices.
|
||||
*/
|
||||
async function isDeviceReady(name: string): Promise<boolean> {
|
||||
try {
|
||||
await (await openFile(`/dev/${name}`, 'r')).close()
|
||||
return true
|
||||
} catch (error) {
|
||||
if (error instanceof Error && 'code' in error && (error.code === 'ENXIO' || error.code === 'ENOENT')) {
|
||||
return false
|
||||
}
|
||||
throw error
|
||||
return Number.parseInt(await readFile(`/sys/class/block/${name}/size`, 'utf8'), 10) > 0
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,11 +54,6 @@ export default class AbstractNbdClient {
|
||||
// publish itself
|
||||
#connectGeneration = 0
|
||||
|
||||
// generation of the attempt which published the current transport, undefined
|
||||
// when not connected: an error on a transport of another generation is stale
|
||||
// and must be ignored
|
||||
#connectedGeneration
|
||||
|
||||
#waitBeforeReconnect
|
||||
#readBlockRetries
|
||||
#reconnectRetry
|
||||
@@ -72,6 +67,7 @@ export default class AbstractNbdClient {
|
||||
#nextCommandQueryId = BigInt(0)
|
||||
// map of command waiting for a response queryId => { size/*in byte*/, resolve, reject}
|
||||
#commandQueryBacklog = new Map()
|
||||
#connected = false
|
||||
|
||||
#reconnectingPromise
|
||||
|
||||
@@ -112,7 +108,7 @@ export default class AbstractNbdClient {
|
||||
}
|
||||
|
||||
get connected() {
|
||||
return this.#connectedGeneration !== undefined
|
||||
return this.#connected
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
@@ -206,31 +202,22 @@ export default class AbstractNbdClient {
|
||||
* connection
|
||||
* ------------------------------------------------------------------------- */
|
||||
|
||||
#onTransportError(generation, error) {
|
||||
#onTransportError = error => {
|
||||
// without this listener an error on the transport would be thrown as an
|
||||
// uncaught exception, and pending reads would only fail on message timeout
|
||||
//
|
||||
// an error on a transport which is not the current one is expected (a
|
||||
// closed transport can be reset late by the other end, after a reconnection)
|
||||
// and must not reject the reads of the current transport
|
||||
//
|
||||
// an error during the handshake is reported by the handshake reads
|
||||
// themselves, through the connect() rejection
|
||||
if (generation !== this.#connectedGeneration) {
|
||||
debug('error on a stale nbd transport', { error })
|
||||
return
|
||||
}
|
||||
warn('error on the nbd transport', { error })
|
||||
// an error outside of the connected window is expected (the other end can
|
||||
// be gone already while we're closing), and always reported to the caller
|
||||
// through the connect()/readBlock() rejection
|
||||
const log = this.#connected ? warn : debug
|
||||
log('error on the nbd transport', { error })
|
||||
this.#rejectAll(error)
|
||||
}
|
||||
|
||||
// the generation is used instead of the transport itself since an attempt can
|
||||
// own multiple transports (the raw one and the one upgraded to TLS)
|
||||
#watchTransport(transport, generation) {
|
||||
const onError = error => this.#onTransportError(generation, error)
|
||||
transport.readable.on('error', onError)
|
||||
#watchTransport(transport) {
|
||||
transport.readable.on('error', this.#onTransportError)
|
||||
if (transport.writable !== transport.readable) {
|
||||
transport.writable.on('error', onError)
|
||||
transport.writable.on('error', this.#onTransportError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,10 +227,10 @@ export default class AbstractNbdClient {
|
||||
// not cancel on timeout, so a superseded attempt must not be able to use,
|
||||
// replace or destroy the transport of the attempt which replaced it
|
||||
let transport = await this._openTransport()
|
||||
this.#watchTransport(transport, generation)
|
||||
this.#watchTransport(transport)
|
||||
try {
|
||||
// the transport can be replaced during the handshake (TLS upgrade)
|
||||
transport = await this.#handshake(transport, generation)
|
||||
transport = await this.#handshake(transport)
|
||||
|
||||
if (generation !== this.#connectGeneration) {
|
||||
const error = new Error('this connection has been superseded by a newer one')
|
||||
@@ -257,7 +244,7 @@ export default class AbstractNbdClient {
|
||||
throw error
|
||||
}
|
||||
this.#transport = transport
|
||||
this.#connectedGeneration = generation
|
||||
this.#connected = true
|
||||
// reset internal state if we reconnected a nbd client
|
||||
this.#commandQueryBacklog = new Map()
|
||||
this.#waitingForResponse = false
|
||||
@@ -280,10 +267,10 @@ export default class AbstractNbdClient {
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
if (this.#connectedGeneration === undefined) {
|
||||
if (!this.#connected) {
|
||||
return
|
||||
}
|
||||
this.#connectedGeneration = undefined
|
||||
this.#connected = false
|
||||
const transport = this.#transport
|
||||
this.#transport = undefined
|
||||
if (transport === undefined) {
|
||||
@@ -379,10 +366,9 @@ export default class AbstractNbdClient {
|
||||
//
|
||||
/**
|
||||
* @param {NbdTransport} transport
|
||||
* @param {number} generation - the connection attempt owning the transport
|
||||
* @returns {Promise<NbdTransport>}
|
||||
*/
|
||||
async #handshake(transport, generation) {
|
||||
async #handshake(transport) {
|
||||
assert((await this.#read(transport, 8)).equals(INIT_PASSWD))
|
||||
assert((await this.#read(transport, 8)).equals(OPTS_MAGIC))
|
||||
const flagsBuffer = await this.#read(transport, 2)
|
||||
@@ -393,7 +379,7 @@ export default class AbstractNbdClient {
|
||||
// let the subclass upgrade the transport if it needs to (TLS)
|
||||
const secured = await this._secureTransport(transport)
|
||||
if (secured !== transport) {
|
||||
this.#watchTransport(secured, generation)
|
||||
this.#watchTransport(secured)
|
||||
transport = secured
|
||||
}
|
||||
|
||||
|
||||
@@ -166,28 +166,6 @@ describe('AbstractNbdClient', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('does not let an error on a superseded transport disturb the current one', async () => {
|
||||
const client = new InMemoryNbdClient({}, { readBlockRetries: 1, waitBeforeReconnect: 0 })
|
||||
await client.connect()
|
||||
await client.reconnect()
|
||||
try {
|
||||
assert.equal(client.transports.length, 2)
|
||||
const [supersededTransport] = client.transports
|
||||
|
||||
// the read is queued synchronously, the in memory server answers later
|
||||
const promise = client.readBlock(0, BLOCK_SIZE)
|
||||
// the other end of the closed transport resets it late
|
||||
supersededTransport.readable.emit('error', Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))
|
||||
|
||||
const block = await promise
|
||||
assert.ok(block.equals(DATA.subarray(0, BLOCK_SIZE)))
|
||||
// no reconnection has been triggered
|
||||
assert.equal(client.transports.length, 2)
|
||||
} finally {
|
||||
await client.disconnect()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not support getMap()', async () => {
|
||||
const client = new InMemoryNbdClient()
|
||||
await assert.rejects(client.getMap(), ({ code }) => code === 'NBD_MAP_UNSUPPORTED')
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import { once } from 'node:events'
|
||||
import { request as httpRequest } from 'node:http'
|
||||
import { request as httpsRequest } from 'node:https'
|
||||
import { isIPv6, Socket } from 'node:net'
|
||||
import { Socket } from 'node:net'
|
||||
import { connect } from 'node:tls'
|
||||
import { spawn } from 'node:child_process'
|
||||
import AbstractNbdClient from './AbstractNbdClient.mjs'
|
||||
@@ -14,9 +11,6 @@ import { NBD_DEFAULT_PORT, NBD_OPT_STARTTLS } from './constants.mjs'
|
||||
* @extends {AbstractNbdClient}
|
||||
*/
|
||||
export default class NbdTcpClient extends AbstractNbdClient {
|
||||
#httpProxy
|
||||
#proxyRejectUnauthorized
|
||||
#proxyTimeout
|
||||
#serverAddress
|
||||
#serverCert
|
||||
#serverPort
|
||||
@@ -27,103 +21,18 @@ export default class NbdTcpClient extends AbstractNbdClient {
|
||||
* @param {number} [settings.port]
|
||||
* @param {string} [settings.exportname]
|
||||
* @param {string} [settings.cert] - PEM certificate, enables TLS when set
|
||||
* @param {string} [settings.httpProxy] - URL of an HTTP(S) proxy, the connection is tunneled through it with `CONNECT`
|
||||
* @param {boolean} [settings.proxyRejectUnauthorized=true] - whether to check the certificate of an HTTPS proxy
|
||||
* @param {number} [settings.proxyTimeout=6e4] - delay in ms after which an unanswered `CONNECT` is aborted
|
||||
* @param {object} [options] - see {@link AbstractNbdClient}
|
||||
*/
|
||||
constructor(
|
||||
{
|
||||
address,
|
||||
port = NBD_DEFAULT_PORT,
|
||||
exportname,
|
||||
cert,
|
||||
httpProxy,
|
||||
proxyRejectUnauthorized = true,
|
||||
proxyTimeout = 6e4,
|
||||
},
|
||||
options
|
||||
) {
|
||||
constructor({ address, port = NBD_DEFAULT_PORT, exportname, cert }, options) {
|
||||
super({ exportname }, options)
|
||||
// other proxy protocols (e.g. SOCKS) are not supported for now
|
||||
if (httpProxy !== undefined) {
|
||||
const { protocol } = new URL(httpProxy)
|
||||
if (protocol !== 'http:' && protocol !== 'https:') {
|
||||
const error = new Error(`unsupported proxy protocol ${protocol}, only http: and https: are supported`)
|
||||
error.code = 'NBD_PROXY_UNSUPPORTED_PROTOCOL'
|
||||
throw error
|
||||
}
|
||||
}
|
||||
this.#serverAddress = address
|
||||
this.#serverPort = port
|
||||
this.#serverCert = cert
|
||||
this.#httpProxy = httpProxy
|
||||
this.#proxyRejectUnauthorized = proxyRejectUnauthorized
|
||||
this.#proxyTimeout = proxyTimeout
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a raw TCP tunnel to the NBD server through an HTTP proxy
|
||||
*
|
||||
* https://datatracker.ietf.org/doc/html/rfc9110#section-9.3.6
|
||||
*
|
||||
* @returns {Promise<import('node:net').Socket>}
|
||||
*/
|
||||
async #connectThroughHttpProxy() {
|
||||
const proxy = new URL(this.#httpProxy)
|
||||
const isHttps = proxy.protocol === 'https:'
|
||||
|
||||
const address = this.#serverAddress
|
||||
const target = `${isIPv6(address) ? `[${address}]` : address}:${this.#serverPort}`
|
||||
const headers = { host: target }
|
||||
if (proxy.username !== '' || proxy.password !== '') {
|
||||
const credentials = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`
|
||||
headers['proxy-authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`
|
||||
}
|
||||
|
||||
const req = (isHttps ? httpsRequest : httpRequest)({
|
||||
agent: false,
|
||||
headers,
|
||||
// URL keeps the brackets around IPv6 addresses
|
||||
hostname: proxy.hostname.replace(/^\[(.*)\]$/, '$1'),
|
||||
method: 'CONNECT',
|
||||
path: target,
|
||||
port: proxy.port,
|
||||
rejectUnauthorized: this.#proxyRejectUnauthorized,
|
||||
timeout: this.#proxyTimeout,
|
||||
})
|
||||
// `timeout` only emits an event, the request must be aborted explicitly
|
||||
req.on('timeout', () => {
|
||||
const error = new Error(`HTTP proxy did not answer the CONNECT to ${target} in time`)
|
||||
error.code = 'NBD_PROXY_CONNECT_TIMEOUT'
|
||||
req.destroy(error)
|
||||
})
|
||||
req.end()
|
||||
|
||||
// the NBD server may already have sent some data, it is in `head`
|
||||
const [res, socket, head] = await once(req, 'connect')
|
||||
// the tunnel is established, NBD has its own timeouts
|
||||
socket.setTimeout(0)
|
||||
if (res.statusCode !== 200) {
|
||||
socket.destroy()
|
||||
const error = new Error(`HTTP proxy refused to connect to ${target}: ${res.statusCode} ${res.statusMessage}`)
|
||||
error.code = 'NBD_PROXY_CONNECT_FAILED'
|
||||
throw error
|
||||
}
|
||||
if (head.length !== 0) {
|
||||
socket.unshift(head)
|
||||
}
|
||||
return socket
|
||||
}
|
||||
|
||||
// mandatory , at least to start the handshake: the connection always starts
|
||||
// unsecured, and is upgraded to TLS during the handshake
|
||||
async _openTransport() {
|
||||
if (this.#httpProxy !== undefined) {
|
||||
const socket = await this.#connectThroughHttpProxy()
|
||||
return { readable: socket, writable: socket }
|
||||
}
|
||||
|
||||
const socket = new Socket()
|
||||
await new Promise((resolve, reject) => {
|
||||
socket.connect(this.#serverPort, this.#serverAddress)
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { after, before, describe, it } from 'node:test'
|
||||
import { createServer as createHttpServer } from 'node:http'
|
||||
import { connect, createServer as createTcpServer } from 'node:net'
|
||||
import { once } from 'node:events'
|
||||
|
||||
import NbdTcpClient from './NbdTcpClient.mjs'
|
||||
import { serveNbd } from './tests/fake-nbd-server.mjs'
|
||||
|
||||
const BLOCK_SIZE = 64 * 1024
|
||||
const DATA = Buffer.alloc(4 * BLOCK_SIZE)
|
||||
for (let i = 0; i < DATA.length; i++) {
|
||||
DATA[i] = i % 251
|
||||
}
|
||||
const CREDENTIALS = 'Basic ' + Buffer.from('user:p@ss').toString('base64')
|
||||
|
||||
describe('NbdTcpClient through an HTTP proxy', () => {
|
||||
let nbdServer, proxyServer, nbdPort, proxyPort
|
||||
const connectRequests = []
|
||||
|
||||
before(async () => {
|
||||
nbdServer = createTcpServer(socket => {
|
||||
serveNbd({ readable: socket, writable: socket, data: DATA }).catch(() => {})
|
||||
})
|
||||
nbdServer.listen(0, '127.0.0.1')
|
||||
await once(nbdServer, 'listening')
|
||||
nbdPort = nbdServer.address().port
|
||||
|
||||
// minimal CONNECT proxy, similar to the XO Proxy one
|
||||
proxyServer = createHttpServer()
|
||||
proxyServer.on('connect', (req, clientSocket, head) => {
|
||||
connectRequests.push(req.url)
|
||||
if (req.headers['proxy-authorization'] !== CREDENTIALS) {
|
||||
clientSocket.end('HTTP/1.1 407 Proxy Authentication Required\r\n\r\n')
|
||||
return
|
||||
}
|
||||
const { hostname, port } = new URL('http://' + req.url)
|
||||
const serverSocket = connect(port, hostname, () => {
|
||||
clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
|
||||
serverSocket.write(head)
|
||||
serverSocket.pipe(clientSocket).pipe(serverSocket)
|
||||
})
|
||||
serverSocket.on('error', () => clientSocket.destroy())
|
||||
clientSocket.on('error', () => serverSocket.destroy())
|
||||
})
|
||||
proxyServer.listen(0, '127.0.0.1')
|
||||
await once(proxyServer, 'listening')
|
||||
proxyPort = proxyServer.address().port
|
||||
})
|
||||
|
||||
after(() => {
|
||||
proxyServer.closeAllConnections()
|
||||
proxyServer.close()
|
||||
nbdServer.close()
|
||||
})
|
||||
|
||||
it('tunnels the NBD connection with CONNECT', async () => {
|
||||
const client = new NbdTcpClient({
|
||||
address: '127.0.0.1',
|
||||
port: nbdPort,
|
||||
httpProxy: `http://user:p%40ss@127.0.0.1:${proxyPort}`,
|
||||
})
|
||||
await client.connect()
|
||||
try {
|
||||
assert.equal(client.exportSize, BigInt(DATA.length))
|
||||
const block = await client.readBlock(2, BLOCK_SIZE)
|
||||
assert.deepEqual(block, DATA.subarray(2 * BLOCK_SIZE, 3 * BLOCK_SIZE))
|
||||
} finally {
|
||||
await client.disconnect()
|
||||
}
|
||||
assert.equal(connectRequests.at(-1), `127.0.0.1:${nbdPort}`)
|
||||
})
|
||||
|
||||
it('fails when the proxy refuses the connection', async () => {
|
||||
const client = new NbdTcpClient({
|
||||
address: '127.0.0.1',
|
||||
port: nbdPort,
|
||||
httpProxy: `http://user:wrong@127.0.0.1:${proxyPort}`,
|
||||
})
|
||||
await assert.rejects(client.connect(), { code: 'NBD_PROXY_CONNECT_FAILED' })
|
||||
await client.disconnect().catch(() => {})
|
||||
})
|
||||
|
||||
it('aborts the CONNECT when the proxy does not answer', async () => {
|
||||
// accepts the TCP connection but never answers
|
||||
const silentProxy = createTcpServer(() => {})
|
||||
silentProxy.listen(0, '127.0.0.1')
|
||||
await once(silentProxy, 'listening')
|
||||
try {
|
||||
const client = new NbdTcpClient({
|
||||
address: '127.0.0.1',
|
||||
port: nbdPort,
|
||||
httpProxy: `http://127.0.0.1:${silentProxy.address().port}`,
|
||||
proxyTimeout: 50,
|
||||
})
|
||||
await assert.rejects(client.connect(), { code: 'NBD_PROXY_CONNECT_TIMEOUT' })
|
||||
} finally {
|
||||
silentProxy.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects unsupported proxy protocols', () => {
|
||||
assert.throws(() => new NbdTcpClient({ address: '127.0.0.1', httpProxy: 'socks5://127.0.0.1:1080' }), {
|
||||
code: 'NBD_PROXY_UNSUPPORTED_PROTOCOL',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -13,7 +13,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "3.7.0",
|
||||
"version": "3.6.0",
|
||||
"engines": {
|
||||
"node": ">=14.0"
|
||||
},
|
||||
@@ -22,10 +22,10 @@
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@vates/read-chunk": "^1.2.1",
|
||||
"@xen-orchestra/async-map": "^0.1.5",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4",
|
||||
"@xen-orchestra/disk-transform": "^1.3.3",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"xen-api": "^5.1.0"
|
||||
"xen-api": "^5.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tap": "^18.7.0",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@vates/types",
|
||||
"private": false,
|
||||
"version": "2.5.0",
|
||||
"version": "2.3.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": {
|
||||
@@ -11,10 +11,6 @@
|
||||
"./xo": "./dist/xo.d.mts",
|
||||
"./xo-app": "./dist/xo-app.d.mts",
|
||||
"./xen-api": "./dist/xen-api.d.mts",
|
||||
"./backup-expression-context": {
|
||||
"default": "./dist/backup-expression-context.mjs",
|
||||
"types": "./dist/backup-expression-context.d.mts"
|
||||
},
|
||||
"./common": {
|
||||
"default": "./dist/common.mjs",
|
||||
"types": "./dist/common.d.mts"
|
||||
@@ -38,7 +34,7 @@
|
||||
"typescript": "~5.6.3",
|
||||
"typescript-eslint": "^8.61.0",
|
||||
"@types/express": "^5.0.0",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4"
|
||||
"@xen-orchestra/disk-transform": "^1.3.3"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
export const BACKUP_EXPRESSION_CONTEXT = {
|
||||
run: {
|
||||
dayOfWeek: { type: 'number' },
|
||||
dayOfMonth: { type: 'number' },
|
||||
hour: { type: 'number' },
|
||||
month: { type: 'number' },
|
||||
year: { type: 'number' },
|
||||
},
|
||||
sr: {
|
||||
name_label: { type: 'string' },
|
||||
type: { type: 'string' },
|
||||
description: { type: 'string' },
|
||||
tags: { type: 'string[]' },
|
||||
},
|
||||
backupRepository: {
|
||||
name: { type: 'string' },
|
||||
type: { type: 'string' },
|
||||
tags: { type: 'string[]' },
|
||||
},
|
||||
vm: {
|
||||
name_label: { type: 'string' },
|
||||
tags: { type: 'string[]' },
|
||||
power_state: { type: 'string' },
|
||||
},
|
||||
chainLength: { type: 'number' },
|
||||
} as const
|
||||
|
||||
export type BACKUP_EXPRESSION_CONTEXT = (typeof BACKUP_EXPRESSION_CONTEXT)[keyof typeof BACKUP_EXPRESSION_CONTEXT]
|
||||
|
||||
export const BACKUP_SETTING_CONTEXT = {
|
||||
healthCheckVmsWithTags: ['vm', 'run'],
|
||||
fullInterval: ['vm', 'run', 'chainLength'],
|
||||
vmFilter: ['vm', 'run'],
|
||||
srs: ['sr', 'run'],
|
||||
remotes: ['backupRepository', 'run'],
|
||||
} as const
|
||||
|
||||
export type BACKUP_SETTING_CONTEXT = (typeof BACKUP_SETTING_CONTEXT)[keyof typeof BACKUP_SETTING_CONTEXT]
|
||||
@@ -663,19 +663,6 @@ export const SUPPORTED_VDI_FORMAT = {
|
||||
|
||||
export type SUPPORTED_VDI_FORMAT = (typeof SUPPORTED_VDI_FORMAT)[keyof typeof SUPPORTED_VDI_FORMAT]
|
||||
|
||||
/**
|
||||
* Formats a VDI can be exported to by XO: the XAPI ones, plus the ones XO generates
|
||||
* itself from the blocks of the disk.
|
||||
*/
|
||||
export const SUPPORTED_VDI_EXPORT_FORMAT = {
|
||||
raw: 'raw',
|
||||
vhd: 'vhd',
|
||||
qcow2: 'qcow2',
|
||||
vmdk: 'vmdk',
|
||||
} as const
|
||||
|
||||
export type SUPPORTED_VDI_EXPORT_FORMAT = (typeof SUPPORTED_VDI_EXPORT_FORMAT)[keyof typeof SUPPORTED_VDI_EXPORT_FORMAT]
|
||||
|
||||
// ----- XAPI Stats
|
||||
|
||||
type XapiStatsResponse<T> = {
|
||||
@@ -772,7 +759,6 @@ export type RawTrafficRule = {
|
||||
ipRange: string
|
||||
direction: TrafficRuleDirection
|
||||
port?: string
|
||||
priority?: number
|
||||
}
|
||||
|
||||
type BaseTrafficRule = RawTrafficRule & {
|
||||
@@ -814,7 +800,3 @@ export const XAPI_TYPES: readonly string[] = [
|
||||
export const RPU_RECOVERY_STEP_NAMES = ['evacuate', 'update', 'reboot', 'enable', 'restoreVms'] as const
|
||||
|
||||
export type RPU_RECOVERY_STEP_NAME = (typeof RPU_RECOVERY_STEP_NAMES)[number]
|
||||
|
||||
export const COMMON_TAG = {
|
||||
NOBAK: 'xo:no-bak',
|
||||
} as const
|
||||
|
||||
@@ -4,8 +4,5 @@ export * from './xo.mjs'
|
||||
export * from './xo-app.mjs'
|
||||
export * from './lib/xen-orchestra-xapi.mjs'
|
||||
export * from './lib/xen-orchestra-acl.mjs'
|
||||
export * from './lib/xen-orchestra-plugins.mjs'
|
||||
export * from './lib/xen-orchestra-v2v.mjs'
|
||||
export * from './lib/complex-matcher.mjs'
|
||||
export * from './lib/rest-api.mjs'
|
||||
export * from './lib/xen-orchestra-vmware-explorer.mjs'
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,219 +0,0 @@
|
||||
import type { Branded } from '../common.mjs'
|
||||
import type { XoNetwork, XoSr, XoVmTemplate } from '../xo.mjs'
|
||||
|
||||
// Stateless API: requests needing the source carry the connection in a POST body (never in URLs or access logs) and
|
||||
// xo-server stores nothing it reads from the source.
|
||||
|
||||
export const V2V_PLATFORM = {
|
||||
VMWARE: 'vmware',
|
||||
} as const
|
||||
|
||||
export type V2V_PLATFORM = (typeof V2V_PLATFORM)[keyof typeof V2V_PLATFORM]
|
||||
|
||||
// 422 rather than 401 for invalid source credentials: a 401 would open the browser Basic authentication prompt
|
||||
export const V2V_ERROR_CODE = {
|
||||
INVALID_CREDENTIALS: 'V2V_INVALID_CREDENTIALS',
|
||||
} as const
|
||||
|
||||
export type V2V_ERROR_CODE = (typeof V2V_ERROR_CODE)[keyof typeof V2V_ERROR_CODE]
|
||||
|
||||
// Clones, restores and replicas inherit `other_config`: the imported VM records its own UUID to be told apart from
|
||||
// them, and the source identity because a moref is only unique within one vCenter. The source identity is the
|
||||
// inventory `instanceUuid`, or `<lowercased host>:<port>` for a standalone ESXi which has none.
|
||||
export const V2V_VM_OTHER_CONFIG = {
|
||||
SOURCE_VM_ID: 'sourceVmId',
|
||||
SOURCE_SNAPSHOT_ID: 'sourceSnapshotId',
|
||||
IMPORTED_VM_UUID: 'xo:v2v:importedVmUuid',
|
||||
SOURCE_ID: 'xo:v2v:sourceId',
|
||||
SOURCE_VM_INSTANCE_UUID: 'xo:v2v:sourceVmInstanceUuid',
|
||||
} as const
|
||||
|
||||
export type V2V_VM_OTHER_CONFIG = (typeof V2V_VM_OTHER_CONFIG)[keyof typeof V2V_VM_OTHER_CONFIG]
|
||||
|
||||
export type XoV2vVmOtherConfig = Partial<Record<V2V_VM_OTHER_CONFIG, string>>
|
||||
|
||||
// The disk of the source chain an imported VDI last received, one of the `chainPaths` of its source disk: the CID
|
||||
// cannot tell a fresh snapshot delta from its parent, the path can
|
||||
export const V2V_VDI_OTHER_CONFIG = {
|
||||
DISK_PATH: 'esxi_diskPath',
|
||||
} as const
|
||||
|
||||
export type V2V_VDI_OTHER_CONFIG = (typeof V2V_VDI_OTHER_CONFIG)[keyof typeof V2V_VDI_OTHER_CONFIG]
|
||||
|
||||
export const V2V_START_BLOCKED_REASON = 'Esxi migration in progress...'
|
||||
|
||||
export const V2V_PARTIAL_IMPORT_TAG = 'xo:no-bak=V2V partial import'
|
||||
|
||||
export const V2V_SNAPSHOT_NAME = {
|
||||
AFTER_PARTIAL_IMPORT: 'after partial import from V2V',
|
||||
AFTER_COMPLETE_IMPORT: 'after complete import from V2V',
|
||||
COMPLETE_IMPORT: 'complete import from V2V',
|
||||
} as const
|
||||
|
||||
export type V2V_SNAPSHOT_NAME = (typeof V2V_SNAPSHOT_NAME)[keyof typeof V2V_SNAPSHOT_NAME]
|
||||
|
||||
// Set on the task of a start so the front can tell which source VMs are being imported
|
||||
export const V2V_TASK_PROPERTY = {
|
||||
SOURCE_VM_IDS: 'sourceVmIds',
|
||||
} as const
|
||||
|
||||
export type XoV2vConnection = {
|
||||
host: string
|
||||
port: number
|
||||
username: string
|
||||
password: string
|
||||
skipSslCheck: boolean
|
||||
}
|
||||
|
||||
export type XoV2vStatus = {
|
||||
vectura: { status: 'success' } | { status: 'error'; error: string }
|
||||
}
|
||||
|
||||
export type XoV2vInventoryPayload = {
|
||||
platform: V2V_PLATFORM
|
||||
connection: XoV2vConnection
|
||||
}
|
||||
|
||||
export type XoV2vDatacenter = {
|
||||
id: Branded<'v2v-datacenter'>
|
||||
name: string
|
||||
}
|
||||
|
||||
export type XoV2vCluster = {
|
||||
id: Branded<'v2v-cluster'>
|
||||
name: string
|
||||
datacenterId: XoV2vDatacenter['id']
|
||||
}
|
||||
|
||||
export type XoV2vHost = {
|
||||
id: Branded<'v2v-host'>
|
||||
name: string
|
||||
clusterId?: XoV2vCluster['id']
|
||||
datacenterId: XoV2vDatacenter['id']
|
||||
}
|
||||
|
||||
// Sizes are in bytes, as in the rest of XO, not in the KB/MB VMware returns
|
||||
export type XoV2vDatastore = {
|
||||
id: Branded<'v2v-datastore'>
|
||||
name: string
|
||||
type: string
|
||||
capacity: number
|
||||
free: number
|
||||
hostIds: XoV2vHost['id'][]
|
||||
}
|
||||
|
||||
export type XoV2vPortGroup = {
|
||||
id: Branded<'v2v-port-group'>
|
||||
name: string
|
||||
vlan?: number
|
||||
}
|
||||
|
||||
export const V2V_DISK_MODE = {
|
||||
PERSISTENT: 'persistent',
|
||||
NON_PERSISTENT: 'nonpersistent',
|
||||
INDEPENDENT_PERSISTENT: 'independent_persistent',
|
||||
INDEPENDENT_NON_PERSISTENT: 'independent_nonpersistent',
|
||||
UNDOABLE: 'undoable',
|
||||
APPEND: 'append',
|
||||
} as const
|
||||
|
||||
export type V2V_DISK_MODE = (typeof V2V_DISK_MODE)[keyof typeof V2V_DISK_MODE]
|
||||
|
||||
// `key` is the vmx device node (`scsi0:0`, `ethernet0`…), the identifier the import matches the start maps against
|
||||
export type XoV2vDisk = {
|
||||
key: Branded<'v2v-disk'>
|
||||
label: string
|
||||
datastoreId: XoV2vDatastore['id']
|
||||
capacity: number
|
||||
mode: V2V_DISK_MODE
|
||||
isRdm: boolean
|
||||
isEncrypted: boolean
|
||||
// `dir/disk.vmdk` in its datastore (no `[datastore]` prefix) of the base disk, then of each snapshot delta down to
|
||||
// the active disk: the chain the import walks for this vmx node
|
||||
chainPaths: string[]
|
||||
}
|
||||
|
||||
// No port group for opaque (NSX) or missing backings
|
||||
export type XoV2vNic = {
|
||||
key: Branded<'v2v-nic'>
|
||||
label: string
|
||||
portGroupId?: XoV2vPortGroup['id']
|
||||
macAddress: string
|
||||
}
|
||||
|
||||
export const V2V_VM_POWER_STATE = {
|
||||
POWERED_ON: 'poweredOn',
|
||||
POWERED_OFF: 'poweredOff',
|
||||
SUSPENDED: 'suspended',
|
||||
} as const
|
||||
|
||||
export type V2V_VM_POWER_STATE = (typeof V2V_VM_POWER_STATE)[keyof typeof V2V_VM_POWER_STATE]
|
||||
|
||||
export const V2V_FIRMWARE = {
|
||||
BIOS: 'bios',
|
||||
UEFI: 'uefi',
|
||||
} as const
|
||||
|
||||
export type V2V_FIRMWARE = (typeof V2V_FIRMWARE)[keyof typeof V2V_FIRMWARE]
|
||||
|
||||
export type XoV2vVm = {
|
||||
id: Branded<'v2v-vm'>
|
||||
instanceUuid: string
|
||||
name: string
|
||||
description: string
|
||||
powerState: V2V_VM_POWER_STATE
|
||||
hostId: XoV2vHost['id']
|
||||
nCpus: number
|
||||
memory: number
|
||||
firmware: V2V_FIRMWARE
|
||||
secureBoot: boolean
|
||||
datastoreIds: XoV2vDatastore['id'][]
|
||||
snapshotCount: number
|
||||
// Compared with `other_config.sourceSnapshotId` of a partial import: a delta is only possible while they match
|
||||
currentSnapshotId?: string
|
||||
disks: XoV2vDisk[]
|
||||
nics: XoV2vNic[]
|
||||
}
|
||||
|
||||
export type XoV2vInventory = {
|
||||
product: string
|
||||
version: string
|
||||
// Absent on a standalone ESXi, identified by its connection address instead
|
||||
instanceUuid?: string
|
||||
datacenters: XoV2vDatacenter[]
|
||||
clusters: XoV2vCluster[]
|
||||
hosts: XoV2vHost[]
|
||||
datastores: XoV2vDatastore[]
|
||||
portGroups: XoV2vPortGroup[]
|
||||
vms: XoV2vVm[]
|
||||
}
|
||||
|
||||
export type XoV2vStartOptions = {
|
||||
stopSource: boolean
|
||||
startVm: boolean
|
||||
cleanup: boolean
|
||||
}
|
||||
|
||||
// A resumed import sends no disk or NIC map: the back end reads its mapping from the existing XCP-ng VM
|
||||
export type XoV2vStartPayloadVm = Partial<XoV2vStartOptions> & {
|
||||
id: XoV2vVm['id']
|
||||
disks?: Record<XoV2vDisk['key'], XoSr['id']>
|
||||
nics?: Record<XoV2vNic['key'], XoNetwork['id']>
|
||||
template?: XoVmTemplate['id']
|
||||
tags: string[]
|
||||
keepMac: boolean
|
||||
}
|
||||
|
||||
// Body of `POST /pools/{id}/actions/import_vms_from_vmware`: the pool is in the path. Checked against the body of the
|
||||
// route when the REST API is built, which refuses any extra key
|
||||
export type XoV2vStartPayload = {
|
||||
connection: XoV2vConnection
|
||||
defaults: XoV2vStartOptions & {
|
||||
sr: XoSr['id']
|
||||
network: XoNetwork['id']
|
||||
template: XoVmTemplate['id']
|
||||
}
|
||||
vms: XoV2vStartPayloadVm[]
|
||||
stopOnError: boolean
|
||||
concurrency: number
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
// shape of `Esxi#getInventory()` in `@xen-orchestra/vmware-explorer`
|
||||
// the objects reference each other through their VMware managed object references (moref)
|
||||
|
||||
export type VmwareDatacenter = {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
export type VmwareCluster = {
|
||||
id: string
|
||||
name: string
|
||||
datacenter?: string
|
||||
}
|
||||
|
||||
export type VmwareHost = {
|
||||
id: string
|
||||
name: string
|
||||
/** a cluster, or the compute resource of a standalone host */
|
||||
parent?: string
|
||||
datacenter?: string
|
||||
}
|
||||
|
||||
export type VmwareDatastore = {
|
||||
id: string
|
||||
name?: string
|
||||
type?: string
|
||||
capacity?: number
|
||||
freeSpace?: number
|
||||
/** the hosts which mount the datastore and can reach it */
|
||||
hosts: string[]
|
||||
}
|
||||
|
||||
export type VmwareVlan = number | { start: number; end: number }[]
|
||||
|
||||
export type VmwareStandardNetwork = {
|
||||
id: string
|
||||
name: string
|
||||
distributed: false
|
||||
}
|
||||
|
||||
export type VmwareDistributedPortgroup = {
|
||||
id: string
|
||||
name: string
|
||||
distributed: true
|
||||
vlan?: VmwareVlan
|
||||
}
|
||||
|
||||
export type VmwareNetwork = VmwareStandardNetwork | VmwareDistributedPortgroup
|
||||
|
||||
export type VmwareVmDisk = {
|
||||
/** node of the disk, as the vmx names it, e.g. `scsi0:0` */
|
||||
key: string
|
||||
label: string
|
||||
capacity: number
|
||||
datastore?: string
|
||||
diskMode?: string
|
||||
rdm: boolean
|
||||
encrypted: boolean
|
||||
/** `dir/disk.vmdk` in its datastore of the base disk, then of each snapshot delta down to the active disk */
|
||||
chainPaths: string[]
|
||||
}
|
||||
|
||||
export type VmwareVmNic = {
|
||||
/** device key of the card */
|
||||
key: string
|
||||
label: string
|
||||
macAddress?: string
|
||||
addressType?: string
|
||||
network?: string
|
||||
}
|
||||
|
||||
export type VmwareVm = {
|
||||
id: string
|
||||
name?: string
|
||||
annotation?: string
|
||||
instanceUuid?: string
|
||||
firmware?: string
|
||||
secureBoot: boolean
|
||||
isTemplate: boolean
|
||||
cpus?: number
|
||||
memory?: number
|
||||
powerState?: 'poweredOff' | 'poweredOn' | 'suspended'
|
||||
host?: string
|
||||
datastores: string[]
|
||||
currentSnapshot?: string
|
||||
snapshotCount: number
|
||||
disks: VmwareVmDisk[]
|
||||
nics: VmwareVmNic[]
|
||||
}
|
||||
|
||||
export type VmwareInventory = {
|
||||
/** e.g. `VMware vCenter Server`, `VMware ESXi` */
|
||||
product?: string
|
||||
version?: string
|
||||
/** only set by a vCenter */
|
||||
instanceUuid?: string
|
||||
datacenters: VmwareDatacenter[]
|
||||
clusters: VmwareCluster[]
|
||||
hosts: VmwareHost[]
|
||||
datastores: VmwareDatastore[]
|
||||
networks: VmwareNetwork[]
|
||||
vms: VmwareVm[]
|
||||
}
|
||||
@@ -130,8 +130,6 @@ export type XsPatches = {
|
||||
uuid?: string
|
||||
}
|
||||
|
||||
type WrappedRecordOfType<Type extends WrappedXenApiRecord['$type']> = Extract<WrappedXenApiRecord, { $type: Type }>
|
||||
|
||||
export interface Xapi {
|
||||
status: string
|
||||
pool?: { uuid: string }
|
||||
@@ -149,14 +147,6 @@ export interface Xapi {
|
||||
ref: T['$ref'],
|
||||
field: K
|
||||
): Promise<T[K]>
|
||||
// the record is inferred from `type`, for a JS caller which cannot spell it out as a type argument
|
||||
getField<Type extends WrappedXenApiRecord['$type'], Field extends keyof WrappedRecordOfType<Type>>(
|
||||
type: Type,
|
||||
ref: WrappedRecordOfType<Type>['$ref'],
|
||||
field: Field
|
||||
): Promise<WrappedRecordOfType<Type>[Field]>
|
||||
/** Synchronous: the record as the object cache currently holds it. */
|
||||
getObjectByRef<Ref extends WrappedXenApiRecord['$ref']>(ref: Ref): Extract<WrappedXenApiRecord, { $ref: Ref }>
|
||||
getObject: <
|
||||
XoRecord extends XapiXoRecord,
|
||||
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
|
||||
@@ -293,42 +283,6 @@ export interface Xapi {
|
||||
}
|
||||
): Promise<XenApiVdi['$ref']>
|
||||
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
|
||||
VBD_create(params: {
|
||||
bootable?: boolean
|
||||
currently_attached?: boolean
|
||||
device?: string
|
||||
empty?: boolean
|
||||
mode?: VBD_MODE
|
||||
other_config?: Record<string, string>
|
||||
qos_algorithm_params?: Record<string, string>
|
||||
qos_algorithm_type?: string
|
||||
/** a plug failure, only logged otherwise, is thrown: the VBD exists all the same */
|
||||
throwVbdPlug?: boolean
|
||||
type?: VBD_TYPE
|
||||
unpluggable?: boolean
|
||||
userdevice?: string
|
||||
VDI?: XenApiVdi['$ref']
|
||||
VM: XenApiVm['$ref']
|
||||
}): Promise<XenApiVbd['$ref']>
|
||||
/** unplugs the VBD first, if needed */
|
||||
VBD_destroy(ref: XenApiVbd['$ref']): Promise<void>
|
||||
VDI_create(
|
||||
params: {
|
||||
name_description?: string
|
||||
name_label: string
|
||||
other_config?: Record<string, string>
|
||||
read_only?: boolean
|
||||
sharable?: boolean
|
||||
SR?: XenApiSr['$ref']
|
||||
tags?: string[]
|
||||
type?: VDI_TYPE
|
||||
virtual_size: number
|
||||
xenstore_data?: Record<string, string>
|
||||
},
|
||||
opts?: { sm_config?: Record<string, string> }
|
||||
): Promise<XenApiVdi['$ref']>
|
||||
/** a VDI which no longer exists counts as destroyed */
|
||||
VDI_destroy(ref: XenApiVdi['$ref']): Promise<void>
|
||||
editVm(
|
||||
id: XoVm['id'],
|
||||
props: EditVmProps,
|
||||
|
||||
@@ -824,10 +824,7 @@ export interface XenApiSr {
|
||||
VDIs: XenApiVdi['$ref'][]
|
||||
virtual_allocation: number
|
||||
}
|
||||
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR', XenApiSrCallMethods> & {
|
||||
// resolved by xen-api from `PBDs`
|
||||
$PBDs: XenApiPbdWrapped[]
|
||||
}
|
||||
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR', XenApiSrCallMethods>
|
||||
|
||||
export interface XenApiSrStat {
|
||||
$ref: Branded<'sr_stat'>
|
||||
|
||||
@@ -1,37 +1,27 @@
|
||||
import type { EventEmitter, Readable } from 'node:stream'
|
||||
import { EventEmitter } from 'node:stream'
|
||||
import type {
|
||||
AnyXoJob,
|
||||
AnyXoLog,
|
||||
NonXapiXoRecord,
|
||||
XoBackupDiskPartition,
|
||||
XapiXoRecord,
|
||||
XoAuthenticationToken,
|
||||
XoBackupRepository,
|
||||
XoConfigBackupArchive,
|
||||
XoGroup,
|
||||
XoHost,
|
||||
XoMetadataBackupJob,
|
||||
XoMirrorBackupJob,
|
||||
XoNetwork,
|
||||
XoPlugin,
|
||||
XoPool,
|
||||
XoPoolBackupArchive,
|
||||
XoProxy,
|
||||
XoSchedule,
|
||||
XoServer,
|
||||
XoSr,
|
||||
XoTask,
|
||||
XoUser,
|
||||
XoVif,
|
||||
XoVm,
|
||||
XoVmBackupArchive,
|
||||
XoVmBackupJob,
|
||||
XoVmTemplate,
|
||||
} from './xo.mjs'
|
||||
import { VatesTask } from './lib/vates-task.mjs'
|
||||
import type { PluginRestRouteDefinition } from './lib/rest-api.mjs'
|
||||
import type { VmwareInventory } from './lib/xen-orchestra-vmware-explorer.mjs'
|
||||
import type { XoV2vStatus } from './lib/xen-orchestra-v2v.mjs'
|
||||
import type { RPU_RECOVERY_STEP_NAME } from './common.mjs'
|
||||
import {
|
||||
Xapi,
|
||||
@@ -185,13 +175,13 @@ export type PoolRollingUpdateRecoveryBlocked = {
|
||||
|
||||
export type PoolRollingUpdateRecovery = PoolRollingUpdateRecoveryRun | PoolRollingUpdateRecoveryBlocked
|
||||
|
||||
/** A disk of a backup archive currently served as an iSCSI LUN */
|
||||
/** A disk of a backup archive currently served as a read-only iSCSI LUN */
|
||||
export type BackupArchiveDiskMount = {
|
||||
/** Handle to pass to `unmountBackupArchiveDisk` */
|
||||
id: string
|
||||
/** UUID of the SR introduced on the host */
|
||||
srUuid: string
|
||||
/** UUID of the VDI exposing the backup disk, read-only unless it is cached */
|
||||
/** UUID of the read-only VDI exposing the backup disk */
|
||||
vdiUuid: string
|
||||
/** IQN of the target serving the disk */
|
||||
iqn: string
|
||||
@@ -201,25 +191,7 @@ export type BackupArchiveDiskMount = {
|
||||
port: number
|
||||
}
|
||||
|
||||
type XoStoreRangeOptions = {
|
||||
gt?: string
|
||||
gte?: string
|
||||
lt?: string
|
||||
lte?: string
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
// `@xen-orchestra/vmware-explorer` is not typed
|
||||
export type EsxiConnectionParams = {
|
||||
/** `host` or `host:port` */
|
||||
host: string
|
||||
user: string
|
||||
password: string
|
||||
sslVerify: boolean
|
||||
}
|
||||
|
||||
export type XoApp = EventEmitter & {
|
||||
export type XoApp = {
|
||||
hooks: EventEmitter
|
||||
_redis: {
|
||||
get(key: string): Promise<string | null>
|
||||
@@ -274,7 +246,6 @@ export type XoApp = EventEmitter & {
|
||||
user?: XoUser
|
||||
permission?: XoUser['permission'] | 'none' | null
|
||||
}
|
||||
backupLogsEe: EventEmitter
|
||||
|
||||
// methods ------------
|
||||
allocIpAddresses(vifId: XoVif['id'], addAddresses?: string[], removeAddresses?: string[]): Promise<void>
|
||||
@@ -299,8 +270,6 @@ export type XoApp = EventEmitter & {
|
||||
backupGuard(objectId: XapiXoRecord['id'], opts?: { bypassBackupCheck?: boolean; operation: string }): Promise<void>
|
||||
/* Throw if no authorization */
|
||||
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
|
||||
/* whether this machine can run the tools of an import from VMware */
|
||||
checkEsxiDependencies(): Promise<XoV2vStatus>
|
||||
/* validate, apply and persist the configuration of a plugin */
|
||||
configurePlugin(id: string, configuration: unknown, mergeWithExisting?: boolean): Promise<void>
|
||||
/* connect a server (XCP-ng/XenServer) */
|
||||
@@ -316,29 +285,7 @@ export type XoApp = EventEmitter & {
|
||||
id: XoAclRole['id'],
|
||||
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
|
||||
): Promise<XoAclRole['id']>
|
||||
createAclV2Role(role: {
|
||||
name: XoAclRole['name']
|
||||
description?: XoAclRole['description']
|
||||
privileges?: Omit<XoAclBasePrivilege, 'id' | 'roleId'>[]
|
||||
}): Promise<XoAclRole>
|
||||
createApiConnection(remoteAddress: string): EventEmitter & {
|
||||
close(): void
|
||||
get<T = unknown>(key: string, defaultValue: unknown): T
|
||||
has(key: string): boolean
|
||||
set(key: string, value: unknown): void
|
||||
unset(key: string): void
|
||||
}
|
||||
createBackupNgJob<T extends XoVmBackupJob | XoMirrorBackupJob>(
|
||||
type: T['type'],
|
||||
props: Omit<T, 'id' | 'type'>,
|
||||
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
|
||||
schedules?: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
|
||||
): Promise<T>
|
||||
createMetadataBackupJob(
|
||||
props: Omit<XoMetadataBackupJob, 'id' | 'type'>,
|
||||
/** keyed by temporary IDs, which can be used as keys in `props.settings` */
|
||||
schedules: Record<string, Pick<XoSchedule, 'cron' | 'enabled' | 'name' | 'timezone'>>
|
||||
): Promise<XoMetadataBackupJob>
|
||||
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
|
||||
createAuthenticationToken(opts: {
|
||||
client?: {
|
||||
id?: string
|
||||
@@ -354,14 +301,6 @@ export type XoApp = EventEmitter & {
|
||||
proxy?: XoProxy['id']
|
||||
url: string
|
||||
}): Promise<XoBackupRepository>
|
||||
createSchedule(params: {
|
||||
cron: XoSchedule['cron']
|
||||
enabled: XoSchedule['enabled']
|
||||
jobId: XoSchedule['jobId']
|
||||
name?: XoSchedule['name']
|
||||
timezone?: XoSchedule['timezone']
|
||||
userId?: XoUser['id']
|
||||
}): Promise<XoSchedule>
|
||||
createUser(params: {
|
||||
firstname?: string
|
||||
lastname?: string
|
||||
@@ -378,14 +317,11 @@ export type XoApp = EventEmitter & {
|
||||
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
|
||||
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
|
||||
deleteGroup(id: XoGroup['id']): Promise<void>
|
||||
deleteSchedule(id: XoSchedule['id']): Promise<void>
|
||||
deleteUser(id: XoUser['id']): Promise<void>
|
||||
detachHostFromPool(hostId: XoHost['id']): Promise<void>
|
||||
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
|
||||
disablePluginAutoload(id: XoPlugin['id']): Promise<void>
|
||||
/* disconnect a server (XCP-ng/XenServer) */
|
||||
disconnectXenServer(id: XoServer['id']): Promise<void>
|
||||
enablePluginAutoload(id: XoPlugin['id']): Promise<void>
|
||||
findEnabledScheduleSequenceFromSchedule(id: XoSchedule['id']): Promise<XoSchedule | undefined>
|
||||
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
|
||||
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
|
||||
@@ -396,7 +332,6 @@ export type XoApp = EventEmitter & {
|
||||
roleId: XoAclRole['id'],
|
||||
opts?: { bypassAuthorization?: boolean }
|
||||
): Promise<boolean>
|
||||
deleteSchedule(id: XoSchedule['id']): Promise<void>
|
||||
getAclV2GroupRoles(
|
||||
groupId: XoGroup['id'],
|
||||
opts?: { bypassAuthorization?: boolean }
|
||||
@@ -411,14 +346,10 @@ export type XoApp = EventEmitter & {
|
||||
getAllProxies(): Promise<XoProxy[]>
|
||||
getAllJobs<T extends AnyXoJob['type']>(type: T): Promise<Extract<AnyXoJob, { type: T }>[]>
|
||||
getAllJobs(type?: string): Promise<AnyXoJob[]>
|
||||
getPlugin(id: XoPlugin['id']): Promise<XoPlugin>
|
||||
getPlugins(): Promise<XoPlugin[]>
|
||||
getProxy(id: XoProxy['id']): Promise<XoProxy>
|
||||
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
|
||||
getAllRemotes(): Promise<XoBackupRepository[]>
|
||||
removeRemote(id: XoBackupRepository['id']): Promise<void>
|
||||
/** Also deletes the schedules of the job */
|
||||
removeJob(id: AnyXoJob['id']): Promise<void>
|
||||
getAllRemotesInfo(): Promise<
|
||||
Record<
|
||||
XoBackupRepository['id'],
|
||||
@@ -447,10 +378,6 @@ export type XoApp = EventEmitter & {
|
||||
}
|
||||
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
|
||||
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
|
||||
/** Please use `REMOVE_CACHE_ENTRY` const to clear the cache */
|
||||
getBackupNgLogs(removeCacheEntry: {}): void
|
||||
/** Please use `REMOVE_CACHE_ENTRY` const to clear a cache entry */
|
||||
getBackupNgLogs(removeCacheEntry: {}, id: AnyXoLog['id']): void
|
||||
getBackupNgLogsSorted(opts: {
|
||||
after?: number
|
||||
before?: number
|
||||
@@ -468,21 +395,8 @@ export type XoApp = EventEmitter & {
|
||||
) => Record<T['id'], T> | undefined
|
||||
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
|
||||
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
|
||||
getStore(namespace: string): Promise<{
|
||||
/** rejects with an error having `notFound: true` when the key does not exist */
|
||||
get(key: string): Promise<unknown>
|
||||
has(key: string): Promise<boolean>
|
||||
put(key: string, value: unknown): Promise<void>
|
||||
del(key: string): Promise<void>
|
||||
del(key: string, cb: (error?: Error | null) => void): void
|
||||
/** object mode stream of `{ key, value }` */
|
||||
createReadStream(opts?: XoStoreRangeOptions): Readable
|
||||
/** object mode stream of keys */
|
||||
createKeyStream(opts?: XoStoreRangeOptions): Readable
|
||||
}>
|
||||
getUser: (id: XoUser['id'], opts?: { obfuscatePassword?: boolean }) => Promise<XoUser>
|
||||
getUserIdentityFields(): string[]
|
||||
getEsxiInventory(params: EsxiConnectionParams): Promise<VmwareInventory>
|
||||
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
|
||||
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
|
||||
getXapiObject: <T extends XapiXoRecord>(
|
||||
@@ -492,29 +406,9 @@ export type XoApp = EventEmitter & {
|
||||
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
|
||||
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
|
||||
getXenServer(id: XoServer['id']): Promise<XoServer>
|
||||
/** the connection of the pool holding this object, which may not be connected */
|
||||
getXenServerIdByObject<T extends XapiXoRecord>(maybeId: T['id'] | T, type?: T['type'] | T['type'][]): XoServer['id']
|
||||
getXenServerWithCredentials(id: XoServer['id']): Promise<XoServer & { password: string }>
|
||||
getXoEventEmitterByType(type: string): EventEmitter
|
||||
hasFeatureAuthorization(featureCode: string): Promise<boolean>
|
||||
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
|
||||
listBackupNgDiskPartitions(remoteId: XoBackupRepository['id'], diskId: string): Promise<XoBackupDiskPartition[]>
|
||||
listBackupNgPartitionFiles(
|
||||
remoteId: XoBackupRepository['id'],
|
||||
diskId: string,
|
||||
partitionId: string | undefined,
|
||||
path: string
|
||||
// keys ending in "/" are directories; files carry size (bytes) and mtime (epoch ms).
|
||||
// Values may be empty ({}) for directories or when served by an older proxy.
|
||||
): Promise<Record<string, { size?: number; mtime?: number }>>
|
||||
fetchBackupNgPartitionFiles(
|
||||
remoteId: XoBackupRepository['id'],
|
||||
diskId: string,
|
||||
partitionId: string | undefined,
|
||||
paths: string[],
|
||||
format: string
|
||||
): Promise<Readable>
|
||||
loadPlugin(id: XoPlugin['id']): Promise<void>
|
||||
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
|
||||
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
|
||||
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
|
||||
@@ -524,41 +418,19 @@ export type XoApp = EventEmitter & {
|
||||
backupRepositoryIds: XoBackupRepository['id'][],
|
||||
opts?: { _forceRefresh?: boolean; vmId?: XoVm['id'] }
|
||||
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]> | null>>
|
||||
/** imports one VM from VMware, resuming a previous partial import when possible, returns the UUID of the XCP-ng VM */
|
||||
migrationfromEsxi(
|
||||
params: EsxiConnectionParams & {
|
||||
sr: XoSr['id']
|
||||
network: XoNetwork['id']
|
||||
/** overrides `sr`, by disk node (e.g. `scsi0:0`) */
|
||||
disks?: Record<string, XoSr['id']>
|
||||
/** overrides `network`, by NIC device key (e.g. `4000`) */
|
||||
nics?: Record<string, XoNetwork['id']>
|
||||
template: XoVmTemplate['id']
|
||||
stopSource: boolean
|
||||
/** VMware moref of the VM */
|
||||
vm: string
|
||||
}
|
||||
): Promise<XoVm['uuid']>
|
||||
/**
|
||||
* Serve one disk of a backup archive as an iSCSI LUN and attach it to `host`
|
||||
* as an SR, read-only unless `cacheSrId` is set. Undone by `unmountBackupArchiveDisk`.
|
||||
* Serve one disk of a backup archive as a read-only iSCSI LUN and attach it to
|
||||
* `host` as an SR. Undone by `unmountBackupArchiveDisk`.
|
||||
*
|
||||
* The LUN is served by whoever can read the backup repository: this appliance, or the proxy the
|
||||
* repository is linked to.
|
||||
*/
|
||||
mountBackupArchiveDisk(params: {
|
||||
archiveId: XoVmBackupArchive['id']
|
||||
/**
|
||||
* SR of a VDI of the serving appliance, which the disk is materialized into as it is read and
|
||||
* which holds the writes; unset, nothing is cached and the mount is read-only. On a local SR,
|
||||
* the serving appliance must not be migrated while the mount lasts: the VDI cannot follow it
|
||||
*/
|
||||
cacheSrId?: XoSr['id']
|
||||
/** One of the archive's `disks[].id` */
|
||||
diskId: string
|
||||
hostId: XoHost['id']
|
||||
}): Promise<BackupArchiveDiskMount>
|
||||
purgePluginConfiguration(id: XoPlugin['id']): Promise<void>
|
||||
pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
|
||||
/**
|
||||
* Record the live mounts a proxy created by itself, while running a restore: they never went
|
||||
@@ -599,7 +471,6 @@ export type XoApp = EventEmitter & {
|
||||
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
|
||||
runJob(job: AnyXoJob, schedule: XoSchedule): void
|
||||
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
|
||||
testPlugin(id: XoPlugin['id'], data?: unknown): Promise<void>
|
||||
testRemote(
|
||||
id: XoBackupRepository['id']
|
||||
): Promise<
|
||||
@@ -609,26 +480,12 @@ export type XoApp = EventEmitter & {
|
||||
touchXoObject(type: string, id: NonXapiXoRecord['id']): Promise<void>
|
||||
/** Detach a disk mounted by `mountBackupArchiveDisk` and stop serving it */
|
||||
unmountBackupArchiveDisk(id: BackupArchiveDiskMount['id']): Promise<void>
|
||||
unloadPlugin(id: XoPlugin['id']): Promise<void>
|
||||
reclaimSpace(
|
||||
remoteId: XoBackupRepository['id'],
|
||||
opts?: { vmUuid?: XoVm['id']; merge?: boolean; remove?: boolean }
|
||||
): Promise<{ vmUuid: string; success: boolean; merge?: boolean; size?: number; error?: string }[]>
|
||||
/** Remove a server from the DB (XCP-ng/XenServer) */
|
||||
unregisterXenServer(id: XoServer['id']): Promise<void>
|
||||
/**
|
||||
* Shallow merge of the properties into the existing job, a `null` value removes the property.
|
||||
*/
|
||||
updateJob(job: { id: AnyXoJob['id']; [key: string]: unknown }, merge?: boolean): Promise<void>
|
||||
updateSchedule(params: {
|
||||
id: XoSchedule['id']
|
||||
cron?: XoSchedule['cron']
|
||||
enabled?: XoSchedule['enabled']
|
||||
jobId?: XoSchedule['jobId']
|
||||
name?: XoSchedule['name']
|
||||
timezone?: XoSchedule['timezone']
|
||||
userId?: XoUser['id']
|
||||
}): Promise<void>
|
||||
updateUser(
|
||||
id: XoUser['id'],
|
||||
updates: {
|
||||
|
||||
@@ -21,29 +21,6 @@ import type {
|
||||
import type * as CMType from './lib/complex-matcher.mjs'
|
||||
import { XoAclBasePrivilege, XoAclRole } from './lib/xen-orchestra-acl.mjs'
|
||||
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
|
||||
import type {
|
||||
XoAuditPlugin,
|
||||
XoAuthGithubPlugin,
|
||||
XoAuthGooglePlugin,
|
||||
XoAuthLdapPlugin,
|
||||
XoAuthOidcPlugin,
|
||||
XoAuthSamlPlugin,
|
||||
XoBackupReportsPlugin,
|
||||
XoIpmiSensorsPlugin,
|
||||
XoLoadBalancerPlugin,
|
||||
XoNetboxPlugin,
|
||||
XoOpenmetricsPlugin,
|
||||
XoPerfAlertPlugin,
|
||||
XoSdnControllerPlugin,
|
||||
XoTestPluginPlugin,
|
||||
XoTransportEmailPlugin,
|
||||
XoTransportIcinga2Plugin,
|
||||
XoTransportNagiosPlugin,
|
||||
XoTransportSlackPlugin,
|
||||
XoTransportXmppPlugin,
|
||||
XoUsageReportPlugin,
|
||||
XoWebHooksPlugin,
|
||||
} from './lib/xen-orchestra-plugins.mjs'
|
||||
|
||||
type BaseXapiXo = {
|
||||
$pool: XoPool['id']
|
||||
@@ -141,7 +118,7 @@ type BaseXoVm = BaseXapiXo & {
|
||||
/**
|
||||
* @deprecated use pvDriversVersion instead
|
||||
*/
|
||||
xenTools?:
|
||||
xentools?:
|
||||
| false
|
||||
| {
|
||||
major: null | number
|
||||
@@ -180,15 +157,6 @@ export type XoBackupLog = BaseXoLog & {
|
||||
export type XoRestoreLog = BaseXoLog & {
|
||||
message: 'restore'
|
||||
}
|
||||
export type XoBackupDiskPartition = {
|
||||
id: Branded<'xo-backup-disk-partition'>
|
||||
name?: string
|
||||
size: number
|
||||
start: number
|
||||
/** number for MBR, string for GPT partition */
|
||||
type: string | number
|
||||
}
|
||||
|
||||
export type XoVmBackupArchive = {
|
||||
id: Branded<'vm-backup-archive'>
|
||||
type: 'xo-vm-backup'
|
||||
@@ -240,7 +208,6 @@ export type XoBackupRepositoryBenchmark = {
|
||||
}
|
||||
|
||||
export type XoBackupRepository = {
|
||||
available?: number
|
||||
benchmarks?: XoBackupRepositoryBenchmark[]
|
||||
enabled: boolean
|
||||
error?: Record<string, unknown>
|
||||
@@ -248,9 +215,7 @@ export type XoBackupRepository = {
|
||||
name: string
|
||||
options?: string
|
||||
proxy?: XoProxy['id']
|
||||
size?: number
|
||||
url: string
|
||||
used?: number
|
||||
}
|
||||
|
||||
export type XoGpuGroup = BaseXapiXo & {
|
||||
@@ -492,29 +457,6 @@ export type XoPool = BaseXapiXo & {
|
||||
zstdSupported: boolean
|
||||
}
|
||||
|
||||
export type XoPlugin =
|
||||
| XoAuditPlugin
|
||||
| XoAuthGithubPlugin
|
||||
| XoAuthGooglePlugin
|
||||
| XoAuthLdapPlugin
|
||||
| XoAuthOidcPlugin
|
||||
| XoAuthSamlPlugin
|
||||
| XoBackupReportsPlugin
|
||||
| XoIpmiSensorsPlugin
|
||||
| XoLoadBalancerPlugin
|
||||
| XoNetboxPlugin
|
||||
| XoOpenmetricsPlugin
|
||||
| XoPerfAlertPlugin
|
||||
| XoSdnControllerPlugin
|
||||
| XoTestPluginPlugin
|
||||
| XoTransportEmailPlugin
|
||||
| XoTransportIcinga2Plugin
|
||||
| XoTransportNagiosPlugin
|
||||
| XoTransportSlackPlugin
|
||||
| XoTransportXmppPlugin
|
||||
| XoUsageReportPlugin
|
||||
| XoWebHooksPlugin
|
||||
|
||||
export type XoProxy = {
|
||||
id: Branded<'proxy'>
|
||||
url: string
|
||||
@@ -539,22 +481,15 @@ type XoBackupJobGeneralSettings = {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps the last backup of each of the last `retention` days/weeks/months/years, in addition to `exportRetention`.
|
||||
*
|
||||
* Requires the `timezone` setting.
|
||||
*/
|
||||
export type XoBackupJobLongTermRetention = {
|
||||
daily?: { retention: number; settings: Record<string, unknown> }
|
||||
weekly?: { retention: number; settings: Record<string, unknown> }
|
||||
monthly?: { retention: number; settings: Record<string, unknown> }
|
||||
yearly?: { retention: number; settings: Record<string, unknown> }
|
||||
}
|
||||
|
||||
export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
cbtDestroySnapshotData?: boolean
|
||||
concurrency?: number
|
||||
longTermRetention?: XoBackupJobLongTermRetention
|
||||
longTermRetention?: {
|
||||
daily?: { retention: number; settings: Record<string, unknown> }
|
||||
weekly?: { retention: number; settings: Record<string, unknown> }
|
||||
monthly?: { retention: number; settings: Record<string, unknown> }
|
||||
yearly?: { retention: number; settings: Record<string, unknown> }
|
||||
}
|
||||
checkpointSnapshot?: boolean
|
||||
offlineSnapshot?: boolean
|
||||
maxExportRate?: number
|
||||
@@ -568,8 +503,8 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
}
|
||||
export type XoVmBackupJobScheduleSettings = {
|
||||
exportRetention?: number
|
||||
healthCheckVmsWithTags?: XoVm['tags'] | string
|
||||
fullInterval?: number | string
|
||||
healthCheckVmsWithTags?: XoVm['tags']
|
||||
fullInterval?: number
|
||||
copyRetention?: number
|
||||
snapshotRetention?: number
|
||||
cbtDestroySnapshotData?: boolean
|
||||
@@ -580,9 +515,9 @@ export type XoVmBackupJob = BaseXoJob & {
|
||||
compression?: 'native' | 'zstd' | ''
|
||||
proxy?: XoProxy['id']
|
||||
mode: 'full' | 'delta'
|
||||
remotes?: CMType.IdOr<XoBackupRepository['id']> | string
|
||||
remotes?: CMType.IdOr<XoBackupRepository['id']>
|
||||
vms: CMType.IdOr<XoVm['id']> | Record<string, unknown>
|
||||
srs?: CMType.IdOr<XoSr['id']> | string
|
||||
srs?: CMType.IdOr<XoSr['id']>
|
||||
type: 'backup'
|
||||
settings: {
|
||||
'': XoVmBackupJobGeneralSettings
|
||||
@@ -611,17 +546,16 @@ export type XoMetadataBackupJob = BaseXoJob & {
|
||||
|
||||
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
concurrency?: number
|
||||
longTermRetention?: XoBackupJobLongTermRetention
|
||||
nRetriesVmBackupFailures?: number
|
||||
mergeBackupsSynchronously?: boolean
|
||||
timeout?: number
|
||||
timezone?: string
|
||||
maxExportRate?: number
|
||||
backupReportTpl?: 'compactMjml'
|
||||
reportWhen: 'failure'
|
||||
}
|
||||
export type XoMirrorBackupScheduleSettings = {
|
||||
exportRetention?: number
|
||||
healthCheckVmsWithTags?: XoVm['tags'] | string
|
||||
vmFilter?: string
|
||||
healthCheckVmsWithTags?: XoVm['tags']
|
||||
healthCheckSr?: XoSr['id']
|
||||
[key: string]: unknown
|
||||
}
|
||||
@@ -675,7 +609,6 @@ export type XoServer = {
|
||||
label?: string
|
||||
master?: XoHost['id']
|
||||
poolId?: XoPool['id']
|
||||
poolMembersAddresses?: string[]
|
||||
poolNameDescription?: string
|
||||
poolNameLabel?: string
|
||||
readOnly: boolean
|
||||
@@ -695,7 +628,6 @@ export type XoSr = BaseXapiXo & {
|
||||
current_operations: Record<string, STORAGE_OPERATIONS>
|
||||
id: Branded<'SR'>
|
||||
inMaintenanceMode: boolean
|
||||
is_tools_sr: boolean
|
||||
name_description: string
|
||||
name_label: string
|
||||
other_config: Record<string, string>
|
||||
@@ -764,7 +696,6 @@ export type XoTask = {
|
||||
| 'backup-log'
|
||||
| 'backup-repository'
|
||||
| 'group'
|
||||
| 'plugin'
|
||||
| 'proxy'
|
||||
| 'restore'
|
||||
| 'restore-log'
|
||||
@@ -978,9 +909,7 @@ export type NonXapiXoRecord =
|
||||
| AnyXoBackupArchive
|
||||
| AnyXoJob
|
||||
| AnyXoLog
|
||||
| XoAuthenticationToken
|
||||
| XoGroup
|
||||
| XoPlugin
|
||||
| XoProxy
|
||||
| XoJob
|
||||
| XoBackupRepository
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"main": "dist/index.mjs",
|
||||
"name": "@xen-orchestra/acl",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
|
||||
"version": "1.8.0",
|
||||
"version": "1.7.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
@@ -30,7 +30,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"@vates/types": "^2.5.0",
|
||||
"@vates/types": "^2.3.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.61.0"
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
export default {
|
||||
export: true, // can export the content ( full or file by file )
|
||||
read: true, // can read the metadata of an archive
|
||||
mount: true, // can mount the archive for file level restore
|
||||
read: true,
|
||||
}
|
||||
|
||||
@@ -25,14 +25,4 @@ export default {
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
probe: {
|
||||
nfs: true,
|
||||
zfs: true,
|
||||
hba: true,
|
||||
iscsiiqn: true,
|
||||
iscsilun: true,
|
||||
'iscsi-exists': true,
|
||||
'hba-exists': true,
|
||||
'nfs-exists': true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import { default as pbd } from './pbd.mjs'
|
||||
import { default as pci } from './pci.mjs'
|
||||
import { default as pgpu } from './pgpu.mjs'
|
||||
import { default as pif } from './pif.mjs'
|
||||
import { default as plugin } from './plugin.mjs'
|
||||
import { default as pool } from './pool.mjs'
|
||||
import { default as proxy } from './proxy.mjs'
|
||||
import { default as restoreLog } from './restore-log.mjs'
|
||||
@@ -57,7 +56,6 @@ export const SUPPORTED_ACTIONS_BY_RESOURCE = {
|
||||
pci,
|
||||
pgpu,
|
||||
pif,
|
||||
plugin,
|
||||
pool,
|
||||
proxy,
|
||||
'restore-log': restoreLog,
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
export default {
|
||||
load: true,
|
||||
read: true,
|
||||
test: true,
|
||||
unload: true,
|
||||
update: {
|
||||
autoload: true,
|
||||
configuration: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import assert from 'node:assert'
|
||||
import type { XoUser } from '@vates/types/xo'
|
||||
|
||||
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
|
||||
@@ -41,21 +42,10 @@ export function hasPrivilegeOn<T extends SupportedResource>({
|
||||
}) {
|
||||
// Function that will be called outside of the module
|
||||
// We cannot be sure types are respected
|
||||
if (typeof user?.permission !== 'string') {
|
||||
throw new TypeError('user.permission must be a string')
|
||||
}
|
||||
|
||||
if (typeof action !== 'string') {
|
||||
throw new TypeError('action must be a string')
|
||||
}
|
||||
|
||||
if (typeof resource !== 'string') {
|
||||
throw new TypeError('resource must be a string')
|
||||
}
|
||||
|
||||
if (objects === undefined) {
|
||||
throw new TypeError('objects must be defined')
|
||||
}
|
||||
assert.strictEqual(typeof user?.permission, 'string')
|
||||
assert.strictEqual(typeof action, 'string')
|
||||
assert.strictEqual(typeof resource, 'string')
|
||||
assert.strictEqual(objects === undefined, false)
|
||||
|
||||
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@xen-orchestra/backup-archive",
|
||||
"version": "1.2.0",
|
||||
"version": "1.1.3",
|
||||
"description": "Helper to handle generic backup archives",
|
||||
"license": "ISC",
|
||||
"author": "",
|
||||
@@ -21,25 +21,25 @@
|
||||
"tar-stream": "^3.1.8",
|
||||
"@types/tar-stream": "^3.1.4",
|
||||
"tmp": "^0.2.1",
|
||||
"@vates/types": "^2.5.0"
|
||||
"@vates/types": "^2.3.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/fs": "^4.10.0",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4",
|
||||
"@xen-orchestra/fs": "^4.9.5",
|
||||
"@xen-orchestra/disk-transform": "^1.3.3",
|
||||
"@vates/task": "0.7.0",
|
||||
"golike-defer": "0.5.1",
|
||||
"uuid": "^9.0.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"@vates/async-each": "^1.0.1",
|
||||
"vhd-lib": "^4.17.0"
|
||||
"vhd-lib": "^4.16.2"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "yarn run clean && tsc",
|
||||
"dev": "tsc --watch",
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "yarn run clean && tsc && node --test './dist/**/*.test.mjs'",
|
||||
"test-integration": "tsc && node --test './src/**/*.integ.mjs'",
|
||||
"test": "yarn run clean && tsc && node --test **/*.test.mjs",
|
||||
"test-integration": "tsc && node --test **/*.integ.mjs",
|
||||
"clean": "rimraf dist/"
|
||||
},
|
||||
"private": false,
|
||||
|
||||
@@ -42,13 +42,12 @@ export class RemoteDiskLineage {
|
||||
#childOf: Map<string, string> = new Map()
|
||||
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
|
||||
#activeDiskPaths: Set<string> = new Set()
|
||||
// Interrupted merges: normalized parent path { stateFilePath, chain, step }
|
||||
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[]; step?: 'mergeBlocks' | 'cleanup' }> =
|
||||
new Map()
|
||||
// Interrupted merges: normalized parent path { stateFilePath, chain }
|
||||
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
|
||||
|
||||
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
|
||||
this.#handler = handler
|
||||
this.#vdiDir = normalize(vdiDir)
|
||||
this.#vdiDir = vdiDir
|
||||
this.#opts = opts
|
||||
}
|
||||
|
||||
@@ -267,38 +266,8 @@ export class RemoteDiskLineage {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Read-only walk toward the descendants: true if any of them is still referenced by a backup.
|
||||
// Unlike getUsedChildChainOrDelete, it neither marks disks visited nor schedules deletions.
|
||||
const hasUsedDescendant = (diskPath: string): boolean => {
|
||||
// `seen` guards against a cycle in corrupted headers
|
||||
const seen = new Set<string>()
|
||||
for (let p: string | undefined = diskPath; p !== undefined && !seen.has(p); p = this.#childOf.get(p)) {
|
||||
seen.add(p)
|
||||
if (!orphanDisks.has(p)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Keeps the orphan loop away from a chain that must not be touched this run. Pinned disks are
|
||||
// differencing: every ancestor they read through must survive too, however deep the lineage is
|
||||
// (e.g. after a retention change). The ancestors are merged normally once the chain is
|
||||
// resumed or dropped.
|
||||
const pin = (paths: readonly string[]): void => {
|
||||
for (const path of paths) {
|
||||
// `seen` guards against a cycle in corrupted headers
|
||||
const seen = new Set<string>()
|
||||
for (let p: string | undefined = path; p !== undefined && !seen.has(p); p = this.#parentOf.get(p)) {
|
||||
seen.add(p)
|
||||
visited.add(p)
|
||||
toDelete.delete(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process interrupted merges first so their disks are protected from the orphan loop
|
||||
for (const [parentPath, { stateFilePath, chain: stateChain, step }] of this.#interruptedMerges) {
|
||||
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
|
||||
if (!this.#diskPaths.has(parentPath)) {
|
||||
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
|
||||
if (remove) {
|
||||
@@ -310,85 +279,14 @@ export class RemoteDiskLineage {
|
||||
|
||||
let chain: string[] | undefined
|
||||
if (stateChain !== undefined) {
|
||||
const missing = stateChain.filter(p => !this.#diskPaths.has(p))
|
||||
|
||||
// A merge only removes disks during its cleanup step, so outside of it every disk of the
|
||||
// recorded chain must still be there. Merging what is left would fold the surviving
|
||||
// children into the parent and then rename it onto the chain tip, silently dropping the
|
||||
// blocks of the disks that went missing. Never resume such a chain.
|
||||
if (missing.length > 0 && step !== 'cleanup') {
|
||||
// Each surviving disk is walked on its own: the missing disk breaks the `childOf` links,
|
||||
// so a walk from the parent alone would stop at the gap.
|
||||
const survivors = stateChain.filter(p => this.#diskPaths.has(p))
|
||||
if (survivors.some(hasUsedDescendant)) {
|
||||
// A backup still references a disk downstream: pin the chain so the orphan loop
|
||||
// neither merges it afresh nor deletes it. It is dropped once nothing uses it anymore.
|
||||
this.#opts.logWarn('merge chain lost disks before its cleanup step, refusing to resume', {
|
||||
stateFilePath,
|
||||
parentPath,
|
||||
step,
|
||||
missing,
|
||||
})
|
||||
pin(survivors)
|
||||
continue
|
||||
}
|
||||
|
||||
// Nothing uses the chain anymore (descendants of the missing disk are broken, so never
|
||||
// active): drop the state, the orphan loop below deletes the disks.
|
||||
this.#opts.logWarn('unmergeable merge chain on a fully orphaned lineage', {
|
||||
stateFilePath,
|
||||
parentPath,
|
||||
step,
|
||||
missing,
|
||||
})
|
||||
if (remove) {
|
||||
this.#opts.logInfo('deleting merge state of orphaned lineage', { stateFilePath })
|
||||
await this.#handler.unlink(stateFilePath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
const existing = stateChain.filter(p => this.#diskPaths.has(p))
|
||||
if (existing.length >= 2) {
|
||||
chain = existing
|
||||
} else {
|
||||
// fewer than 2 disks of the recorded chain are left, there is nothing to resume and
|
||||
// the state would otherwise stay on disk forever.
|
||||
//
|
||||
// only done when the chain was read successfully: an undefined `stateChain` also
|
||||
// covers a state file that is merely unreadable right now, which must be kept
|
||||
this.#opts.logWarn('merge state without a resumable chain', { stateFilePath, parentPath })
|
||||
if (remove) {
|
||||
this.#opts.logInfo('deleting unresumable merge state', { stateFilePath })
|
||||
await this.#handler.unlink(stateFilePath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (existing.length >= 2) chain = existing
|
||||
}
|
||||
|
||||
if (chain === undefined) {
|
||||
// state file unreadable: leave it alone, the orphan loop below schedules a fresh merge
|
||||
continue
|
||||
if (chain !== undefined) {
|
||||
chain.forEach(p => visited.add(p))
|
||||
toMerge.push({ chain, isResuming: true })
|
||||
}
|
||||
|
||||
// Nothing downstream of the merge parent is still referenced by a backup: merging would
|
||||
// write into a lineage nothing points at, and pinning it below would exempt it from the
|
||||
// orphan loop forever. The walk already marked the whole lineage for deletion, so only
|
||||
// the state file is left to drop.
|
||||
//
|
||||
// It walks the live lineage rather than `stateChain`, which is frozen at the retention
|
||||
// boundary of the run that created it and can be shorter than the real chain.
|
||||
if (getUsedChildChainOrDelete(parentPath) === undefined) {
|
||||
this.#opts.logWarn('merge state on a fully orphaned lineage', { stateFilePath, parentPath })
|
||||
if (remove) {
|
||||
this.#opts.logInfo('deleting merge state of orphaned lineage', { stateFilePath })
|
||||
await this.#handler.unlink(stateFilePath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
pin(chain)
|
||||
toMerge.push({ chain, isResuming: true })
|
||||
}
|
||||
|
||||
for (const orphan of orphanDisks) {
|
||||
|
||||
@@ -1,279 +0,0 @@
|
||||
import { test } from 'node:test'
|
||||
import { strict as assert } from 'node:assert'
|
||||
|
||||
import tmp from 'tmp'
|
||||
import * as uuid from 'uuid'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { normalize } from '@xen-orchestra/fs/path'
|
||||
import { pFromCallback } from 'promise-toolbox'
|
||||
import { rimraf } from 'rimraf'
|
||||
import { Constants, VhdAbstract, VhdDirectory } from 'vhd-lib'
|
||||
|
||||
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
|
||||
// `RemoteDiskLineage` is a `.mts` module: this is the TS ESM spelling of the sibling import, which
|
||||
// resolves at runtime because the `test` script runs the built `dist/` copy of this file. It must
|
||||
// NOT go through `../dist/`, which would make `tsc` pull the emitted declarations back in as
|
||||
// inputs (TS5055) — the `*.integ.mjs` files can only do that because tsconfig excludes them.
|
||||
/* eslint-disable n/no-missing-import */
|
||||
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
|
||||
/* eslint-enable n/no-missing-import */
|
||||
|
||||
const { beforeEach, afterEach, describe } = test
|
||||
|
||||
let handler, tempDir, vdiDir
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
handler = getHandler({ url: `file://${tempDir}` })
|
||||
await handler.sync()
|
||||
// VmIncrementalBackupArchive passes a normalized path, and RemoteDiskLineage compares
|
||||
// #vdiDir against normalized paths, so the fixture has to match
|
||||
vdiDir = normalize(`xo-vm-backups/VMUUID/vdis/${uuid.v1()}/${uuid.v1()}`)
|
||||
await handler.mktree(`${vdiDir}/data`)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rimraf(tempDir)
|
||||
await handler.forget()
|
||||
})
|
||||
|
||||
// Creates an aliased VHD directory in `vdiDir`, chained onto `parent` when given.
|
||||
async function generateDisk(name, { parent } = {}) {
|
||||
const aliasPath = `${vdiDir}/${name}.alias.vhd`
|
||||
const dataPath = `${vdiDir}/data/${name}.vhd`
|
||||
|
||||
await handler.mkdir(dataPath)
|
||||
const vhd = new VhdDirectory(handler, dataPath)
|
||||
vhd.header = { ...VHDHEADER }
|
||||
vhd.footer = { ...VHDFOOTER, uuid: uuid.v1({}, Buffer.alloc(16)) }
|
||||
if (parent !== undefined) {
|
||||
vhd.header.parentUnicodeName = `${parent.name}.alias.vhd`
|
||||
vhd.header.parentUuid = parent.vhd.footer.uuid
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
|
||||
} else {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
|
||||
}
|
||||
await vhd.writeEntireBlock({ id: 0, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, 1) })
|
||||
await vhd.writeBlockAllocationTable()
|
||||
await vhd.writeHeader()
|
||||
await vhd.writeFooter()
|
||||
await VhdAbstract.createAlias(handler, aliasPath, dataPath)
|
||||
|
||||
return { name, aliasPath, dataPath, vhd }
|
||||
}
|
||||
|
||||
// Plants the merge state file a previous interrupted merge would have left on `parent`.
|
||||
async function writeMergeState(parent, { chain, step = 'mergeBlocks' }) {
|
||||
const statePath = `${vdiDir}/.${parent.name}.alias.vhd.merge.json`
|
||||
await handler.writeFile(
|
||||
statePath,
|
||||
JSON.stringify({
|
||||
child: { uuid: uuid.v1() },
|
||||
parent: { uuid: uuid.v1() },
|
||||
chain: chain.map(name => `${name}.alias.vhd`),
|
||||
currentBlock: 0,
|
||||
mergedDataSize: 0,
|
||||
step,
|
||||
diskSize: 0,
|
||||
})
|
||||
)
|
||||
return statePath
|
||||
}
|
||||
|
||||
async function cleanLineage({ activeDisks = [], lineageDir = vdiDir, merge = false } = {}) {
|
||||
const warnings = []
|
||||
const infos = []
|
||||
const lineage = new RemoteDiskLineage(handler, lineageDir, {
|
||||
remove: true,
|
||||
merge,
|
||||
logInfo: (message, data) => infos.push({ message, data }),
|
||||
logWarn: (message, data) => warnings.push({ message, data }),
|
||||
})
|
||||
await lineage.init()
|
||||
for (const disk of activeDisks) {
|
||||
lineage.addActiveDiskPath(disk.aliasPath)
|
||||
}
|
||||
await lineage.clean({ remove: true, merge })
|
||||
return { warnings, infos }
|
||||
}
|
||||
|
||||
const exists = path => handler.list(vdiDir).then(files => files.includes(path.slice(vdiDir.length + 1)))
|
||||
|
||||
describe('RemoteDiskLineage.clean() interrupted merges', { concurrency: 1 }, () => {
|
||||
test('drops the merge state and the disks when no descendant is still referenced', async () => {
|
||||
const base = await generateDisk('base')
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
|
||||
|
||||
// no active disk: the whole lineage is out of retention
|
||||
const { warnings } = await cleanLineage()
|
||||
|
||||
assert.equal(await exists(statePath), false, 'merge state should be deleted')
|
||||
assert.equal(await exists(base.aliasPath), false, 'base should be deleted')
|
||||
assert.equal(await exists(child.aliasPath), false, 'child should be deleted')
|
||||
assert.ok(
|
||||
warnings.some(({ message }) => message === 'merge state on a fully orphaned lineage'),
|
||||
'the dropped chain should be reported'
|
||||
)
|
||||
})
|
||||
|
||||
test('refuses to resume, and keeps everything, when the chain lost a disk before its cleanup step', async () => {
|
||||
const base = await generateDisk('base')
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
// `missing` was recorded in the chain but is no longer on the remote: merging base + child
|
||||
// and renaming onto the tip would silently drop its blocks
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
|
||||
|
||||
// merge enabled: the orphan loop must not pick the refused chain up and merge it afresh
|
||||
const { warnings, infos } = await cleanLineage({ activeDisks: [child], merge: true })
|
||||
|
||||
assert.equal(
|
||||
infos.some(({ message }) => message === 'Disk chain needs merging'),
|
||||
false,
|
||||
'no merge should be scheduled for a chain that was refused'
|
||||
)
|
||||
assert.equal(await exists(statePath), true, 'merge state should be kept while a backup uses the chain')
|
||||
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
|
||||
const warning = warnings.find(
|
||||
({ message }) => message === 'merge chain lost disks before its cleanup step, refusing to resume'
|
||||
)
|
||||
assert.notEqual(warning, undefined, 'the incomplete chain should be reported')
|
||||
assert.deepEqual(warning.data.missing, [`${vdiDir}/missing.alias.vhd`])
|
||||
})
|
||||
|
||||
test('keeps every ancestor of a chain it refused to resume', async () => {
|
||||
// several ancestors fell out of retention after the merge was interrupted (e.g. the retention
|
||||
// was lowered): deleting any of them would leave `base`, a differencing disk, without a parent
|
||||
const ggp = await generateDisk('ggp')
|
||||
const gp = await generateDisk('gp', { parent: ggp })
|
||||
const base = await generateDisk('base', { parent: gp })
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
|
||||
|
||||
const { warnings, infos } = await cleanLineage({ activeDisks: [child], merge: true })
|
||||
|
||||
assert.ok(
|
||||
warnings.some(({ message }) => message === 'merge chain lost disks before its cleanup step, refusing to resume'),
|
||||
'the incomplete chain should be reported'
|
||||
)
|
||||
assert.equal(
|
||||
infos.some(({ message }) => message === 'Disk chain needs merging'),
|
||||
false,
|
||||
'neither the refused chain nor its ancestors should be merged'
|
||||
)
|
||||
assert.equal(await exists(ggp.aliasPath), true, 'ggp should be kept')
|
||||
assert.equal(await exists(gp.aliasPath), true, 'gp should be kept')
|
||||
assert.equal(await exists(statePath), true, 'merge state should be kept')
|
||||
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
})
|
||||
|
||||
test('keeps every ancestor of a resumed chain, and merges them on the next run', async () => {
|
||||
const ggp = await generateDisk('ggp')
|
||||
const gp = await generateDisk('gp', { parent: ggp })
|
||||
const base = await generateDisk('base', { parent: gp })
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
|
||||
|
||||
// first run: only the interrupted merge is resumed, its ancestors are left alone
|
||||
const firstRun = await cleanLineage({ activeDisks: [child], merge: true })
|
||||
|
||||
assert.ok(
|
||||
firstRun.infos.some(({ message, data }) => message === 'merging disk chain' && data.chain.length === 2),
|
||||
'the interrupted merge should be resumed'
|
||||
)
|
||||
assert.equal(await exists(statePath), false, 'merge state should be deleted once the merge completed')
|
||||
assert.equal(await exists(base.aliasPath), false, 'base should be merged into child')
|
||||
assert.equal(await exists(ggp.aliasPath), true, 'ggp should be kept')
|
||||
assert.equal(await exists(gp.aliasPath), true, 'gp should be kept')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
|
||||
// second run: the ancestors are merged into child as a regular chain
|
||||
const secondRun = await cleanLineage({ activeDisks: [child], merge: true })
|
||||
|
||||
assert.deepEqual(
|
||||
secondRun.warnings.filter(({ message }) => message === 'disk broken: parent missing or broken'),
|
||||
[],
|
||||
'child should still have its parents'
|
||||
)
|
||||
assert.ok(
|
||||
secondRun.infos.some(({ message, data }) => message === 'merging disk chain' && data.chain.length === 3),
|
||||
'ggp and gp should be merged into child'
|
||||
)
|
||||
assert.equal(await exists(ggp.aliasPath), false, 'ggp should be merged into child')
|
||||
assert.equal(await exists(gp.aliasPath), false, 'gp should be merged into child')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
})
|
||||
|
||||
test('drops an unmergeable chain, state and disks, once nothing uses it', async () => {
|
||||
const base = await generateDisk('base')
|
||||
const missing = await generateDisk('missing', { parent: base })
|
||||
const child = await generateDisk('child', { parent: missing })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'missing', 'child'] })
|
||||
// the intermediate disk vanished mid-merge: child is now broken, so its backup is incomplete
|
||||
// and never declares it active
|
||||
await handler.unlink(missing.aliasPath)
|
||||
await handler.rmtree(missing.dataPath)
|
||||
|
||||
const { warnings, infos } = await cleanLineage({ merge: true })
|
||||
|
||||
assert.equal(
|
||||
infos.some(({ message }) => message === 'Disk chain needs merging'),
|
||||
false,
|
||||
'an unmergeable chain should not be merged'
|
||||
)
|
||||
assert.equal(await exists(statePath), false, 'merge state should be deleted')
|
||||
assert.equal(await exists(base.aliasPath), false, 'base should be deleted')
|
||||
assert.equal(await exists(child.aliasPath), false, 'child should be deleted')
|
||||
assert.ok(
|
||||
warnings.some(({ message }) => message === 'unmergeable merge chain on a fully orphaned lineage'),
|
||||
'the dropped chain should be reported'
|
||||
)
|
||||
})
|
||||
|
||||
test('drops a merge state whose chain no longer has two disks to merge, during cleanup', async () => {
|
||||
const base = await generateDisk('base')
|
||||
// the cleanup step unlinks the children as it goes, so a gap there is expected
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'alreadyUnlinked'], step: 'cleanup' })
|
||||
|
||||
const { warnings } = await cleanLineage({ activeDisks: [base] })
|
||||
|
||||
assert.equal(await exists(statePath), false, 'merge state should be deleted')
|
||||
assert.equal(await exists(base.aliasPath), true, 'the active disk should be kept')
|
||||
assert.ok(
|
||||
warnings.some(({ message }) => message === 'merge state without a resumable chain'),
|
||||
'the unresumable state should be reported'
|
||||
)
|
||||
})
|
||||
|
||||
test('keeps a resumable merge state and its disks', async () => {
|
||||
const base = await generateDisk('base')
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
|
||||
|
||||
const { warnings } = await cleanLineage({ activeDisks: [child] })
|
||||
|
||||
assert.equal(await exists(statePath), true, 'merge state should be kept')
|
||||
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
assert.deepEqual(warnings, [], 'a resumable merge should not warn')
|
||||
})
|
||||
|
||||
test('behaves the same when built with a non-normalized VDI directory', async () => {
|
||||
const base = await generateDisk('base')
|
||||
const child = await generateDisk('child', { parent: base })
|
||||
const statePath = await writeMergeState(base, { chain: ['base', 'child'] })
|
||||
|
||||
// the constructor must normalize: otherwise #cleanOrphanDataFiles fails to exclude the VDI
|
||||
// directory from the data directories it sweeps and deletes every unclaimed file in it,
|
||||
// starting with the merge state
|
||||
const { warnings } = await cleanLineage({ activeDisks: [child], lineageDir: vdiDir.replace(/^\//, '') })
|
||||
|
||||
assert.equal(await exists(statePath), true, 'merge state should be kept')
|
||||
assert.equal(await exists(base.aliasPath), true, 'base should be kept')
|
||||
assert.equal(await exists(child.aliasPath), true, 'child should be kept')
|
||||
assert.deepEqual(warnings, [], 'nothing should be reported as orphaned')
|
||||
})
|
||||
})
|
||||
@@ -331,7 +331,7 @@ export class MergeRemoteDisk {
|
||||
* @param {string[]} files - all file paths in vdiDir (prependDir: true)
|
||||
* @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
|
||||
* @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
|
||||
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[], step?: 'mergeBlocks' | 'cleanup' }>>}
|
||||
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
|
||||
*/
|
||||
static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
|
||||
const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
|
||||
@@ -343,12 +343,8 @@ export class MergeRemoteDisk {
|
||||
|
||||
const parentPath = normalize(vdiDir + '/' + match[1])
|
||||
let chain
|
||||
let step
|
||||
try {
|
||||
const state = JSON.parse(await handler.readFile(filePath))
|
||||
// disks are only removed during the cleanup step, so the caller needs the step to tell
|
||||
// an expected gap in the chain from a lineage that lost data
|
||||
step = state?.step ?? 'mergeBlocks'
|
||||
if (Array.isArray(state?.chain)) {
|
||||
chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
|
||||
} else {
|
||||
@@ -361,7 +357,7 @@ export class MergeRemoteDisk {
|
||||
warn("Merge state unreadable, can't restart merging.", error)
|
||||
}
|
||||
|
||||
result.set(parentPath, { stateFilePath: filePath, chain, step })
|
||||
result.set(parentPath, { stateFilePath: filePath, chain })
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
* @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('vhd-lib/disk-consumer/compressedBlock.mjs').CompressedDiskBlock} CompressedDiskBlock
|
||||
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
|
||||
*
|
||||
|
||||
@@ -46,36 +45,26 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
*/
|
||||
#blockSize = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#bitmapSize = 512
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @type {string | undefined}
|
||||
*/
|
||||
#rawBlocksCompression
|
||||
|
||||
/**
|
||||
* set in init(): this disk is a VHD directory using #rawBlocksCompression
|
||||
* @type {boolean}
|
||||
*/
|
||||
#rawBlocks = false
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {RemoteHandlerAbstract} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {string} [params.rawBlocksCompression] if this disk is a VHD directory using this compression, its blocks
|
||||
* are read as stored (see CompressedDiskBlock), to be written as is in a VHD directory using the same compression.
|
||||
* Otherwise (plain VHD, other compression), they are read normally
|
||||
*/
|
||||
constructor({ handler, path, rawBlocksCompression }) {
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
this.#rawBlocksCompression = rawBlocksCompression
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -98,9 +87,6 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
this.#vhd = undefined
|
||||
throw Object.assign(new Error("Can't init vhd directory without using alias"), { code: 'NOT_SUPPORTED' })
|
||||
}
|
||||
// the disk may predate a change of the remote configuration: only use its raw blocks if they match
|
||||
this.#rawBlocks =
|
||||
this.#rawBlocksCompression !== undefined && this.#getCompressionType() === this.#rawBlocksCompression
|
||||
|
||||
if (!options.ignoreBlockIndexes) {
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
@@ -277,7 +263,10 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
await this.#vhd.writeBlockData(diskBlock.index, diskBlock.data)
|
||||
await this.#vhd.writeEntireBlock({
|
||||
id: diskBlock.index,
|
||||
buffer: Buffer.concat([Buffer.alloc(this.#bitmapSize, 255), diskBlock.data]),
|
||||
})
|
||||
|
||||
return this.getBlockSize()
|
||||
}
|
||||
@@ -285,22 +274,12 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock | CompressedDiskBlock>} diskBlock
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
if (this.#rawBlocks) {
|
||||
/** @type {CompressedDiskBlock} */
|
||||
const block = {
|
||||
index,
|
||||
// only a VHD directory has a compression, checked in init()
|
||||
data: await /** @type {VhdDirectory} */ (this.#vhd).readCompressedBlock(index),
|
||||
compression: /** @type {string} */ (this.#rawBlocksCompression),
|
||||
}
|
||||
return block
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@xen-orchestra/async-map": "^0.1.5",
|
||||
"@xen-orchestra/backups": "^0.78.0",
|
||||
"@xen-orchestra/fs": "^4.10.0",
|
||||
"@xen-orchestra/backups": "^0.76.0",
|
||||
"@xen-orchestra/fs": "^4.9.5",
|
||||
"filenamify": "^6.0.0",
|
||||
"getopts": "^2.2.5",
|
||||
"lodash": "^4.18.0",
|
||||
@@ -28,7 +28,7 @@
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"version": "1.1.18",
|
||||
"version": "1.1.16",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
|
||||
@@ -2,7 +2,6 @@ import assert from 'node:assert'
|
||||
|
||||
import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { importIncrementalVm } from './_incrementalVm.mjs'
|
||||
import { liveMountXapiLabels } from './liveMountXapiLabels.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
import { decorateClass } from '@vates/decorate-with'
|
||||
import { Task } from '@vates/task'
|
||||
@@ -43,8 +42,7 @@ export class ImportVmBackup {
|
||||
* @param {object} [params.liveMount] - how to serve a disk from the backup repository instead of
|
||||
* copying it, injected by the caller since a live mount outlives the restore and cannot be run
|
||||
* from this package
|
||||
* @param {({ cacheSrUuid, diskPath, hostId, xapiLabels }) => Promise<{ id: string, vdiUuid: string }>} params.liveMount.mountDisk
|
||||
* `xapiLabels` are those of {@link liveMountXapiLabels}, `cacheSrUuid` makes the mount read/write
|
||||
* @param {({ diskPath, hostId }) => Promise<{ id: string, vdiUuid: string }>} params.liveMount.mountDisk
|
||||
* @param {(id: string) => Promise<void>} params.liveMount.unmountDisk
|
||||
* @param {object} [params.settings]
|
||||
*/
|
||||
@@ -358,23 +356,11 @@ export class ImportVmBackup {
|
||||
}
|
||||
|
||||
const diskPath = join(metadataDir, metadata.vhds[vdiRef])
|
||||
const cacheSrUuid = this._vdiRestoreTargets.get(vdi.uuid).cacheSr
|
||||
const readWrite = cacheSrUuid !== undefined
|
||||
const xapiLabels = liveMountXapiLabels({
|
||||
readWrite,
|
||||
timestamp: metadata.timestamp,
|
||||
vdiNameLabel: vdi.name_label,
|
||||
vmNameLabel: metadata.vm.name_label,
|
||||
})
|
||||
const mount = await liveMount.mountDisk({ cacheSrUuid, diskPath, hostId, xapiLabels })
|
||||
const mount = await liveMount.mountDisk({ diskPath, hostId })
|
||||
this.#liveMounts.push({ ...mount, hostId })
|
||||
info('disk live mounted', { cacheSrUuid, diskPath, hostId, mountId: mount.id, vdiUuid: vdi.uuid })
|
||||
info('disk live mounted', { diskPath, hostId, mountId: mount.id, vdiUuid: vdi.uuid })
|
||||
|
||||
const record = {
|
||||
...vdi,
|
||||
liveMountedVdiRef: await xapi.call('VDI.get_by_uuid', mount.vdiUuid),
|
||||
liveMountedReadOnly: !readWrite,
|
||||
}
|
||||
const record = { ...vdi, liveMountedVdiRef: await xapi.call('VDI.get_by_uuid', mount.vdiUuid) }
|
||||
// it is attached as is: there is nothing to clone it from, nothing to transfer into it, and
|
||||
// no SR to create it on — `SR` still holds the ref it had on the backed up pool
|
||||
delete record.baseVdi
|
||||
|
||||
@@ -141,20 +141,7 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
|
||||
|
||||
const backup = await importer._decorateIncrementalVmMetadata()
|
||||
|
||||
assert.deepEqual(liveMount.mounted, [
|
||||
{
|
||||
// not cached: read only
|
||||
cacheSrUuid: undefined,
|
||||
diskPath: `/xo-vm-backups/${VM_UUID}/${vhdPath(DATA)}`,
|
||||
hostId: HOST,
|
||||
// named after the backed up VM and disk, not after the file on the backup repository
|
||||
xapiLabels: {
|
||||
srNameLabel: '[XO live mount] a vm (20250801T080832Z)',
|
||||
vdiNameLabel: 'data',
|
||||
vdiNameDescription: 'Read-only live mount of a vm (20250801T080832Z), removed on unmount.',
|
||||
},
|
||||
},
|
||||
])
|
||||
assert.deepEqual(liveMount.mounted, [{ diskPath: `/xo-vm-backups/${VM_UUID}/${vhdPath(DATA)}`, hostId: HOST }])
|
||||
// no data is transferred for it
|
||||
assert.deepEqual(Object.keys(backup.disks), [SYSTEM.ref])
|
||||
})
|
||||
@@ -166,7 +153,6 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
|
||||
|
||||
const vdi = backup.vdis[DATA.ref]
|
||||
assert.equal(vdi.liveMountedVdiRef, 'VDI_REF:mounted-vdi-1')
|
||||
assert.equal(vdi.liveMountedReadOnly, true)
|
||||
assert.equal(vdi.SR, undefined)
|
||||
assert.equal(vdi.uuid, DATA.uuid)
|
||||
// the other disk is restored as usual
|
||||
@@ -174,19 +160,6 @@ describe('ImportVmBackup#_decorateIncrementalVmMetadata()', () => {
|
||||
assert.equal(backup.vdis[SYSTEM.ref].liveMountedVdiRef, undefined)
|
||||
})
|
||||
|
||||
it('mounts a cached disk read/write, and says so in its names', async () => {
|
||||
const { importer, liveMount } = makeImporter({
|
||||
mapVdisSrs: { [DATA.uuid]: { type: 'live-mount', host: HOST, cacheSr: OTHER_SR } },
|
||||
})
|
||||
|
||||
const backup = await importer._decorateIncrementalVmMetadata()
|
||||
|
||||
const [mounted] = liveMount.mounted
|
||||
assert.equal(mounted.cacheSrUuid, OTHER_SR)
|
||||
assert.equal(mounted.xapiLabels.srNameLabel, '[XO live restore] a vm (20250801T080832Z)')
|
||||
assert.equal(backup.vdis[DATA.ref].liveMountedReadOnly, false)
|
||||
})
|
||||
|
||||
it('rejects live mounts spread over several hosts', async () => {
|
||||
const { importer } = makeImporter({
|
||||
mapVdisSrs: {
|
||||
|
||||
@@ -40,10 +40,10 @@ afterEach(async () => {
|
||||
const uniqueId = () => uuid.v1()
|
||||
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
|
||||
|
||||
async function getAdapter({ useVhdDirectory = false, remoteCompressionType } = {}) {
|
||||
async function getAdapter({ useVhdDirectory = false, vhdDirectoryCompression, remoteCompressionType } = {}) {
|
||||
handler = getHandler({ url: `file://${tempDir}`, useVhdDirectory, compressionType: remoteCompressionType })
|
||||
await handler.sync()
|
||||
return new RemoteAdapter(handler)
|
||||
return new RemoteAdapter(handler, { vhdDirectoryCompression })
|
||||
}
|
||||
|
||||
const listVmDir = () => handler.list(rootPath, { ignoreMissing: true })
|
||||
@@ -145,8 +145,7 @@ describe('RemoteAdapter#isMergeableParent', { concurrency: 1 }, () => {
|
||||
|
||||
test(`diskIsDirectory=${diskIsDirectory} useVhdDirectory=${useVhdDirectory} compressionMatches=${compressionMatches} uuidMatches=${uuidMatches} -> ${expected}`, async () => {
|
||||
const targetCompression = 'brotli'
|
||||
// no compressionType on the remote: brotli
|
||||
const adapter = await getAdapter({ useVhdDirectory })
|
||||
const adapter = await getAdapter({ useVhdDirectory, vhdDirectoryCompression: targetCompression })
|
||||
const targetUuid = uniqueIdBuffer()
|
||||
|
||||
await generateVhd(`${basePath}/disk.vhd`, {
|
||||
@@ -165,32 +164,37 @@ describe('RemoteAdapter#isMergeableParent', { concurrency: 1 }, () => {
|
||||
})
|
||||
|
||||
describe('RemoteAdapter#isMergeableParent compression resolution', { concurrency: 1 }, () => {
|
||||
test('the compressionType of the remote is the one compared against', async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
|
||||
test('the per-remote compressionType override wins over the adapter default', async () => {
|
||||
const adapter = await getAdapter({
|
||||
useVhdDirectory: true,
|
||||
vhdDirectoryCompression: 'gzip', // config.toml-style default
|
||||
remoteCompressionType: 'brotli', // per-remote override, should be the one actually compared against
|
||||
})
|
||||
const targetUuid = uniqueIdBuffer()
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'gzip', uuid: targetUuid })
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
|
||||
|
||||
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), true)
|
||||
})
|
||||
|
||||
test('a disk compressed with another codec than the remote is rejected', async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
|
||||
test('a disk compressed with the adapter default (not the override) is rejected', async () => {
|
||||
const adapter = await getAdapter({
|
||||
useVhdDirectory: true,
|
||||
vhdDirectoryCompression: 'gzip',
|
||||
remoteCompressionType: 'brotli',
|
||||
})
|
||||
const targetUuid = uniqueIdBuffer()
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
|
||||
// disk uses the config default, not the remote override that's actually in effect
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'gzip', uuid: targetUuid })
|
||||
|
||||
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), false)
|
||||
})
|
||||
|
||||
test('a remote without compressionType uses brotli', async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true })
|
||||
const targetUuid = uniqueIdBuffer()
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', compression: 'brotli', uuid: targetUuid })
|
||||
|
||||
assert.equal(await adapter.isMergeableParent(targetUuid, `${basePath}/disk.vhd.alias.vhd`), true)
|
||||
})
|
||||
|
||||
test("a remote-level 'none' means no compression", async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'none' })
|
||||
test("a remote-level 'none' override means no compression, ignoring the adapter default", async () => {
|
||||
const adapter = await getAdapter({
|
||||
useVhdDirectory: true,
|
||||
vhdDirectoryCompression: 'brotli',
|
||||
remoteCompressionType: 'none',
|
||||
})
|
||||
const targetUuid = uniqueIdBuffer()
|
||||
// no compression option => VhdDirectory's own getCompressionType() is undefined, matching 'none'
|
||||
await generateVhd(`${basePath}/disk.vhd`, { mode: 'directory', uuid: targetUuid })
|
||||
@@ -319,105 +323,3 @@ describe('RemoteAdapter#writeVhd validated by checkDisk', { concurrency: 1 }, ()
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// IncrementalRemote runner when source and targets use the same compression: the blocks are copied as stored
|
||||
describe('RemoteAdapter raw blocks copy', { concurrency: 1 }, () => {
|
||||
// copy a disk the way the IncrementalRemote runner does, and check the target content
|
||||
async function copyAndCheck(adapter, sourcePath, { rawBlocksCompression, expectedCompression }) {
|
||||
const targetPath = `${basePath}/${adapter.getVhdFileName('target')}`
|
||||
const source = await adapter._createVhdDisk(handler, sourcePath, { useChain: false, rawBlocksCompression })
|
||||
await adapter.writeVhd(targetPath, source, {
|
||||
validator: tmpPath => checkDisk(handler, tmpPath),
|
||||
uuid: uniqueIdBuffer(),
|
||||
})
|
||||
|
||||
await Disposable.use(openVhd(handler, targetPath), async vhd => {
|
||||
assert.equal(vhd.compressionType, expectedCompression)
|
||||
await vhd.readBlockAllocationTable()
|
||||
assert.ok(vhd.containsBlock(0))
|
||||
assert.ok(!vhd.containsBlock(1))
|
||||
const { data } = await vhd.readBlock(2)
|
||||
assert.ok(
|
||||
data.every(byte => byte === 0x42),
|
||||
'block 2 should contain the source disk data'
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
for (const compression of ['brotli', 'gzip', 'none']) {
|
||||
test(`${compression}: a disk read with the same compression is copied as stored, readable and identical`, async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: compression })
|
||||
const sourcePath = `${basePath}/${adapter.getVhdFileName('source')}`
|
||||
await adapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
|
||||
|
||||
const source = await adapter._createVhdDisk(handler, sourcePath, {
|
||||
useChain: false,
|
||||
rawBlocksCompression: compression,
|
||||
})
|
||||
const block = await source.readBlock(2)
|
||||
await source.close()
|
||||
assert.equal(block.compression, compression)
|
||||
if (compression !== 'none') {
|
||||
assert.ok(block.data.length < 2 * 1024 * 1024, 'the block should be read compressed')
|
||||
}
|
||||
|
||||
await copyAndCheck(adapter, sourcePath, { rawBlocksCompression: compression, expectedCompression: compression })
|
||||
})
|
||||
}
|
||||
|
||||
test('a disk written before a compression change is read normally and copied', async () => {
|
||||
const gzipAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'gzip' })
|
||||
const sourcePath = `${basePath}/${gzipAdapter.getVhdFileName('source')}`
|
||||
await gzipAdapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
|
||||
await handler.forget()
|
||||
|
||||
const brotliAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
|
||||
const source = await brotliAdapter._createVhdDisk(handler, sourcePath, {
|
||||
useChain: false,
|
||||
rawBlocksCompression: 'brotli',
|
||||
})
|
||||
const block = await source.readBlock(2)
|
||||
await source.close()
|
||||
assert.equal(block.compression, undefined)
|
||||
assert.equal(block.data.length, 2 * 1024 * 1024)
|
||||
|
||||
await copyAndCheck(brotliAdapter, sourcePath, { rawBlocksCompression: 'brotli', expectedCompression: 'brotli' })
|
||||
})
|
||||
|
||||
test('a VHD file written before the switch to VHD directories is read normally and copied', async () => {
|
||||
const fileAdapter = await getAdapter({ useVhdDirectory: false })
|
||||
const sourcePath = `${basePath}/${fileAdapter.getVhdFileName('source')}`
|
||||
await fileAdapter.writeVhd(sourcePath, new MockDisk(3, [0, 2], 0x42), { uuid: uniqueIdBuffer() })
|
||||
await handler.forget()
|
||||
|
||||
const directoryAdapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
|
||||
await copyAndCheck(directoryAdapter, sourcePath, { rawBlocksCompression: 'brotli', expectedCompression: 'brotli' })
|
||||
})
|
||||
|
||||
// blocks as read from a VHD directory using `compression`, without their real content: only the checks are tested
|
||||
class MockCompressedDisk extends MockDisk {
|
||||
#compression
|
||||
constructor(compression, ...args) {
|
||||
super(...args)
|
||||
this.#compression = compression
|
||||
}
|
||||
|
||||
async readBlock(index) {
|
||||
return { ...(await super.readBlock(index)), compression: this.#compression }
|
||||
}
|
||||
}
|
||||
|
||||
test('a VHD directory target refuses blocks compressed differently', async () => {
|
||||
const adapter = await getAdapter({ useVhdDirectory: true, remoteCompressionType: 'brotli' })
|
||||
await assert.rejects(
|
||||
adapter.writeVhd(
|
||||
`${basePath}/${adapter.getVhdFileName('target')}`,
|
||||
new MockCompressedDisk('gzip', 1, [0], 0x42),
|
||||
{
|
||||
uuid: uniqueIdBuffer(),
|
||||
}
|
||||
),
|
||||
/can't write a gzip compressed block 0/
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,7 +3,6 @@ import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { stringify } from 'uuid'
|
||||
import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import assert from 'node:assert'
|
||||
import { basename, dirname, join, resolve } from 'node:path'
|
||||
import { normalize } from '@xen-orchestra/fs/path'
|
||||
import { synchronized } from 'decorator-synchronized'
|
||||
@@ -27,7 +26,6 @@ import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { isMetadataFile } from './_backupType.mjs'
|
||||
import {
|
||||
isKnownJournalEvent,
|
||||
purgeBackupJournal,
|
||||
readBackupJournal,
|
||||
writeBackupJournalEntries,
|
||||
writeBackupJournalEntry,
|
||||
@@ -66,10 +64,6 @@ export class RemoteAdapter {
|
||||
this._handler = handler
|
||||
this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
|
||||
this._useGetDiskLegacy = useGetDiskLegacy
|
||||
|
||||
// Cache of the partition list per diskId, populated by listPartitions and evicted by
|
||||
// getDisk when the disk is unmounted
|
||||
this._partitionsCache = new Map()
|
||||
}
|
||||
|
||||
get handler() {
|
||||
@@ -323,14 +317,6 @@ export class RemoteAdapter {
|
||||
return this.handler.getConfig('useVhdDirectory')
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string | undefined} the compression of the VHD directories written on this remote ('none' if not
|
||||
* compressed), undefined if this remote writes plain VHD files
|
||||
*/
|
||||
getVhdDirectoryCompression() {
|
||||
return this.useVhdDirectory() ? (this.handler.getConfig('compressionType') ?? 'brotli') : undefined // compatibility layer
|
||||
}
|
||||
|
||||
#useAlias() {
|
||||
return this.useVhdDirectory()
|
||||
}
|
||||
@@ -560,14 +546,6 @@ export class RemoteAdapter {
|
||||
return readBackupJournal(this._handler, cursor, opts)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} before timestamp in ms, the journal days before its own are removed
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async purgeBackupJournal(before) {
|
||||
return purgeBackupJournal(this._handler, before)
|
||||
}
|
||||
|
||||
// Same as `readBackupJournal()`, with the current metadata of the added and changed backups
|
||||
// attached, so that a listing can be brought up to date from the result alone, in a single
|
||||
// round-trip for a caller which is not on this host.
|
||||
@@ -698,7 +676,7 @@ export class RemoteAdapter {
|
||||
path,
|
||||
concurrency: writeBlockConcurrency,
|
||||
validator,
|
||||
compression: this.getVhdDirectoryCompression(),
|
||||
compression: handler.getConfig('compressionType') ?? 'brotli', // compatibility layer
|
||||
uuid,
|
||||
parentUuid,
|
||||
parentPath,
|
||||
@@ -724,8 +702,7 @@ export class RemoteAdapter {
|
||||
) {
|
||||
const container = watchStreamSize(input)
|
||||
await this._handler.outputStream(path, input, {
|
||||
// a known checksum is the one of unencrypted data: an encrypted remote stores no checksum
|
||||
checksum: typeof checksum === 'string' && this._handler.isEncrypted ? false : checksum,
|
||||
checksum,
|
||||
dirMode: this._dirMode,
|
||||
maxStreamLength,
|
||||
streamLength,
|
||||
@@ -738,24 +715,19 @@ export class RemoteAdapter {
|
||||
}
|
||||
|
||||
// open the hierarchy of ancestors until we find a full one
|
||||
async _createVhdDisk(handler, path, { useChain, rawBlocksCompression }) {
|
||||
async _createVhdDisk(handler, path, { useChain }) {
|
||||
let disk
|
||||
if (useChain) {
|
||||
assert.strictEqual(rawBlocksCompression, undefined, `can't read raw blocks of a disk chain`)
|
||||
disk = await openDiskChain({ handler, path })
|
||||
} else {
|
||||
disk = new RemoteVhdDisk({ handler, path, rawBlocksCompression })
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
}
|
||||
disk = new ReadAhead(disk)
|
||||
return disk
|
||||
}
|
||||
|
||||
/**
|
||||
* `rawBlocksCompression`: the blocks of the VHD directories using this compression are read as stored (see
|
||||
* RemoteVhdDisk), to be written as is in a remote using the same compression
|
||||
*/
|
||||
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true, rawBlocksCompression } = {}) {
|
||||
async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
|
||||
const handler = this._handler
|
||||
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
|
||||
const dir = dirname(metadata._filename)
|
||||
@@ -763,7 +735,7 @@ export class RemoteAdapter {
|
||||
const disks = {}
|
||||
await asyncMapSettled(Object.keys(vdis), async ref => {
|
||||
delete vdis[ref].baseVdi
|
||||
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain, rawBlocksCompression })
|
||||
disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -781,28 +753,6 @@ export class RemoteAdapter {
|
||||
return this._handler.createReadStream(resolve('/', dirname(metadata._filename), metadata.xva))
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<string | undefined>} the checksum of the XVA as stored, undefined if it is unknown (no checksum
|
||||
* file, or encrypted remote: the stored data are not the data read)
|
||||
*/
|
||||
async readFullVmBackupChecksum(metadata) {
|
||||
if (this._handler.isEncrypted) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
const checksum = String(
|
||||
await this._handler.readFile(resolve('/', dirname(metadata._filename), metadata.xva) + '.checksum')
|
||||
).trim()
|
||||
// $<algorithm id>$<salt>$<hash>
|
||||
return checksum.startsWith('$') ? checksum : undefined
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async readVmBackupMetadata(path) {
|
||||
let json
|
||||
let isImmutable = false
|
||||
|
||||
@@ -393,47 +393,6 @@ describe('readBackupJournal()', { concurrency: 1 }, () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('purgeBackupJournal()', { concurrency: 1 }, () => {
|
||||
const DAY = 24 * 60 * 60 * 1e3
|
||||
const HOUR = 60 * 60 * 1e3
|
||||
|
||||
// midnight of the given number of days ago, so that the tests do not depend on the time they run at
|
||||
const daysAgo = n => (Math.floor(Date.now() / DAY) - n) * DAY
|
||||
|
||||
const writeEntryAt = timestamp =>
|
||||
writeJournalEntryAt(timestamp, { event: 'add', vmUuid, filename: `/${rootPath}/${timestamp}.json` })
|
||||
|
||||
test('removes the days before the one of `before`, and keeps that day whole', async () => {
|
||||
for (const n of [3, 2]) {
|
||||
await writeEntryAt(daysAgo(n) + HOUR)
|
||||
}
|
||||
|
||||
// after the entry of its own day: that entry is kept all the same
|
||||
await adapter.purgeBackupJournal(daysAgo(2) + 2 * HOUR)
|
||||
|
||||
assert.deepEqual(await listJournalDays(), [formatJournalDay(daysAgo(2))])
|
||||
})
|
||||
|
||||
test('skips a day which cannot be removed, and removes the next ones', async t => {
|
||||
for (const n of [3, 2]) {
|
||||
await writeEntryAt(daysAgo(n))
|
||||
}
|
||||
const lockedDay = formatJournalDay(daysAgo(3))
|
||||
|
||||
const rmtree = handler.rmtree.bind(handler)
|
||||
t.mock.method(handler, 'rmtree', async path => {
|
||||
if (path === `/${BACKUP_JOURNAL_DIR}/${lockedDay}`) {
|
||||
throw Object.assign(new Error('EPERM: operation not permitted'), { code: 'EPERM' })
|
||||
}
|
||||
return rmtree(path)
|
||||
})
|
||||
|
||||
await adapter.purgeBackupJournal(Date.now())
|
||||
|
||||
assert.deepEqual(await listJournalDays(), [lockedDay])
|
||||
})
|
||||
})
|
||||
|
||||
describe('readBackupJournalEvents()', { concurrency: 1 }, () => {
|
||||
// records the paths whose metadata is read from now on, to check a replay does not re-read the
|
||||
// same backup once per event
|
||||
|
||||
@@ -309,47 +309,3 @@ export async function readBackupJournal(handler, cursor, { mustExist = false } =
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the journal entries of the days before the one of `before`.
|
||||
*
|
||||
* Whole days are removed, entries are never removed one by one: the day of `before` is kept entirely,
|
||||
* whatever the time of its entries.
|
||||
*
|
||||
* A day which cannot be removed is skipped, e.g. its files are immutable (`EPERM` on a local
|
||||
* repository, `AccessDenied` from the policy of a bucket): it must not keep the later days from being
|
||||
* removed, and the next purge tries it again. Only a failure to list the journal rejects.
|
||||
*
|
||||
* A purge warns once, with its first failure and their count: where deletes are refused, every day
|
||||
* before `before` fails again on each purge, one more each day.
|
||||
*
|
||||
* @param {RemoteHandler} handler
|
||||
* @param {number} before timestamp in ms
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function purgeBackupJournal(handler, before) {
|
||||
const minDay = formatJournalDay(before)
|
||||
const days = await handler.list(`/${BACKUP_JOURNAL_DIR}`, {
|
||||
filter: name => isDayName(name) && name < minDay,
|
||||
ignoreMissing: true,
|
||||
})
|
||||
let firstFailure
|
||||
let nFailures = 0
|
||||
// one day at a time: this runs in the background, it must not compete with the backups for the
|
||||
// remote
|
||||
for (const day of days) {
|
||||
const path = `/${BACKUP_JOURNAL_DIR}/${day}`
|
||||
try {
|
||||
await handler.rmtree(path)
|
||||
debug('journal day purged', { path })
|
||||
} catch (error) {
|
||||
if (firstFailure === undefined) {
|
||||
firstFailure = { error, path }
|
||||
}
|
||||
++nFailures
|
||||
}
|
||||
}
|
||||
if (firstFailure !== undefined) {
|
||||
warn('failed to purge some journal days', { ...firstFailure, nFailures })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ class BackupWorker {
|
||||
yield new RemoteAdapter(handler, {
|
||||
debounceResource: this.debounceResource,
|
||||
dirMode: this.#config.dirMode,
|
||||
vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
|
||||
})
|
||||
} finally {
|
||||
await handler.forget()
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import moment from 'moment-timezone'
|
||||
import { parse } from 'xo-remote-parser'
|
||||
|
||||
const { warn } = createLogger('xo:backups:buildContext')
|
||||
|
||||
/**
|
||||
* @typedef {import('@vates/types/xen-api').XenApiVm} XenApiVm
|
||||
* @typedef {import('@vates/types/xo').XoSr} XoSr
|
||||
* @typedef {import('@vates/types/xo').XoBackupRepository} XoBackupRepository
|
||||
*
|
||||
* @typedef {object} RunContext
|
||||
* @property {number} dayOfWeek
|
||||
* @property {number} dayOfMonth
|
||||
* @property {number} hour
|
||||
* @property {number} month
|
||||
* @property {number} year
|
||||
*
|
||||
* @typedef {object} VmContext
|
||||
* @property {Record<string, unknown>} vm
|
||||
* @property {RunContext} run
|
||||
* @property {number} [chainLength]
|
||||
*
|
||||
* @typedef {object} SrContext
|
||||
* @property {{name_label: string, type: string, description: string, tags: string[]}} sr
|
||||
* @property {RunContext} run
|
||||
*
|
||||
* @typedef {object} BackupRepositoryContext
|
||||
* @property {{name: string, type: string | undefined, tags: string[]}} backupRepository
|
||||
* @property {RunContext} run
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {Date} date
|
||||
* @param {string | undefined} timezone
|
||||
* @returns {RunContext}
|
||||
*/
|
||||
export function buildRunContext(date, timezone) {
|
||||
const momentDate = timezone ? moment.tz(date, timezone) : moment(date)
|
||||
return {
|
||||
dayOfWeek: momentDate.day(),
|
||||
dayOfMonth: momentDate.date(),
|
||||
hour: momentDate.hour(),
|
||||
month: momentDate.month() + 1,
|
||||
year: momentDate.year(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {XenApiVm} vm
|
||||
* @param {RunContext} run
|
||||
* @param {number} [chainLength]
|
||||
* @returns {VmContext}
|
||||
*/
|
||||
export function buildVmContext(vm, run, chainLength) {
|
||||
/** @type {Record<string, unknown>} */
|
||||
const vmObject = {}
|
||||
for (const [key, value] of Object.entries(vm)) {
|
||||
if (!key.startsWith('$') && typeof value !== 'function') {
|
||||
vmObject[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
if (!Array.isArray(vmObject.tags)) {
|
||||
vmObject.tags = []
|
||||
}
|
||||
|
||||
return { vm: vmObject, run, chainLength }
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {XoSr} sr
|
||||
* @param {RunContext} run
|
||||
* @returns {SrContext}
|
||||
*/
|
||||
export function buildSrContext(sr, run) {
|
||||
return {
|
||||
sr: {
|
||||
name_label: sr.name_label,
|
||||
type: sr.SR_type,
|
||||
description: sr.name_description,
|
||||
tags: Array.isArray(sr.tags) ? sr.tags : [],
|
||||
},
|
||||
run,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {XoBackupRepository} backupRepository
|
||||
* @param {RunContext} run
|
||||
* @returns {BackupRepositoryContext}
|
||||
*/
|
||||
export function buildBackupRepositoryContext(backupRepository, run) {
|
||||
let type
|
||||
try {
|
||||
type = parse(backupRepository.url).type
|
||||
} catch (error) {
|
||||
warn('cannot parse backup repository URL', { backupRepositoryId: backupRepository.id, error })
|
||||
}
|
||||
|
||||
return {
|
||||
backupRepository: {
|
||||
name: backupRepository.name,
|
||||
type,
|
||||
tags: [], // Placeholder - Backup repositories don't have tags yet
|
||||
},
|
||||
run,
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { buildRunContext, buildVmContext, buildSrContext, buildBackupRepositoryContext } from './_buildContext.mjs'
|
||||
import { BACKUP_EXPRESSION_CONTEXT } from '@vates/types/backup-expression-context'
|
||||
|
||||
describe('buildContext', () => {
|
||||
const date = new Date('2025-01-05T03:00:00Z')
|
||||
const runContext = buildRunContext(date, 'America/New_York')
|
||||
const expectedRunContext = {
|
||||
dayOfWeek: 6,
|
||||
dayOfMonth: 4,
|
||||
hour: 22,
|
||||
month: 1,
|
||||
year: 2025,
|
||||
}
|
||||
|
||||
it('buildRunContext', () => {
|
||||
assert.deepEqual(runContext, expectedRunContext)
|
||||
})
|
||||
|
||||
it('buildVmContext', () => {
|
||||
const vm1 = { name: 'name', $value: '$value', func: () => {} }
|
||||
const vmContext1 = buildVmContext(vm1, runContext, 3)
|
||||
assert.deepEqual(vmContext1, {
|
||||
vm: {
|
||||
name: 'name',
|
||||
tags: [],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
chainLength: 3,
|
||||
})
|
||||
|
||||
const vm2 = { name: 'name', $value: '$value', func: () => {}, tags: ['tag'] }
|
||||
const vmContext2 = buildVmContext(vm2, runContext, undefined)
|
||||
assert.deepEqual(vmContext2, {
|
||||
vm: {
|
||||
name: 'name',
|
||||
tags: ['tag'],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
chainLength: undefined,
|
||||
})
|
||||
|
||||
const vm3 = { name: 'name', $value: '$value', func: () => {}, tags: 'tag' }
|
||||
const vmContext3 = buildVmContext(vm3, runContext, undefined)
|
||||
assert.deepEqual(vmContext3, {
|
||||
vm: {
|
||||
name: 'name',
|
||||
tags: [],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
chainLength: undefined,
|
||||
})
|
||||
})
|
||||
|
||||
it('buildVmContext exposes every field advertised by the schema', () => {
|
||||
// vm is an open context (buildVmContext copies through all real VM fields),
|
||||
// so we assert the schema's curated fields are a subset of the output, not equality.
|
||||
const vm = { name_label: 'name', power_state: 'Running', tags: ['tag'], $ref: 'ref', func: () => {} }
|
||||
const { vm: vmContext } = buildVmContext(vm, runContext)
|
||||
|
||||
for (const field of Object.keys(BACKUP_EXPRESSION_CONTEXT.vm)) {
|
||||
assert.ok(field in vmContext, `vm context is missing schema field: ${field}`)
|
||||
}
|
||||
})
|
||||
|
||||
it('buildSrContext', () => {
|
||||
const sr1 = {
|
||||
name_label: 'name_label',
|
||||
SR_type: 'type',
|
||||
name_description: 'description',
|
||||
ignored: 'ignored',
|
||||
}
|
||||
const srContext1 = buildSrContext(sr1, runContext)
|
||||
assert.deepEqual(srContext1, {
|
||||
sr: {
|
||||
name_label: 'name_label',
|
||||
type: 'type',
|
||||
description: 'description',
|
||||
tags: [],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
})
|
||||
assert.deepEqual(Object.keys(srContext1.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
|
||||
|
||||
const sr2 = {
|
||||
name_label: 'name_label',
|
||||
SR_type: 'type',
|
||||
name_description: 'description',
|
||||
tags: ['tag'],
|
||||
ignored: 'ignored',
|
||||
}
|
||||
const srContext2 = buildSrContext(sr2, runContext)
|
||||
assert.deepEqual(srContext2, {
|
||||
sr: {
|
||||
name_label: 'name_label',
|
||||
type: 'type',
|
||||
description: 'description',
|
||||
tags: ['tag'],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
})
|
||||
assert.deepEqual(Object.keys(srContext2.sr).sort(), Object.keys(BACKUP_EXPRESSION_CONTEXT.sr).sort())
|
||||
})
|
||||
|
||||
it('buildBackupRepositoryContext', () => {
|
||||
const backupRepository1 = {
|
||||
name: 'name',
|
||||
url: 'nfs://host/path',
|
||||
ignored: 'ignored',
|
||||
}
|
||||
const backupRepositoryContext1 = buildBackupRepositoryContext(backupRepository1, runContext)
|
||||
assert.deepEqual(backupRepositoryContext1, {
|
||||
backupRepository: {
|
||||
name: 'name',
|
||||
type: 'nfs',
|
||||
tags: [],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
})
|
||||
assert.deepEqual(
|
||||
Object.keys(backupRepositoryContext1.backupRepository).sort(),
|
||||
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
|
||||
)
|
||||
|
||||
const backupRepository2 = {
|
||||
name: 'name',
|
||||
url: 'nfs://host/path',
|
||||
tags: [],
|
||||
ignored: 'ignored',
|
||||
}
|
||||
const backupRepositoryContext2 = buildBackupRepositoryContext(backupRepository2, runContext)
|
||||
assert.deepEqual(backupRepositoryContext2, {
|
||||
backupRepository: {
|
||||
name: 'name',
|
||||
type: 'nfs',
|
||||
tags: [],
|
||||
},
|
||||
run: expectedRunContext,
|
||||
})
|
||||
assert.deepEqual(
|
||||
Object.keys(backupRepositoryContext2.backupRepository).sort(),
|
||||
Object.keys(BACKUP_EXPRESSION_CONTEXT.backupRepository).sort()
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,24 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
import * as CM from 'complex-matcher'
|
||||
|
||||
/**
|
||||
* Parse and compile a complex-matcher expression into a predicate function.
|
||||
* Throws synchronously if the expression is syntactically invalid.
|
||||
*
|
||||
* @param {string} expression
|
||||
* @returns {(context: object) => boolean}
|
||||
*/
|
||||
export function compileExpression(expression) {
|
||||
return CM.parse(expression).createPredicate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if value is a non-empty string — i.e. a complex-matcher expression.
|
||||
*
|
||||
* @param {unknown} value
|
||||
* @returns {value is string}
|
||||
*/
|
||||
export function isExpression(value) {
|
||||
return typeof value === 'string' && value.length > 0 && !Number.isFinite(Number(value))
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { compileExpression, isExpression } from './_expressionPredicate.mjs'
|
||||
|
||||
describe('compileExpression', () => {
|
||||
it('run:dayOfWeek:0', () => {
|
||||
const predicate = compileExpression('run:dayOfWeek:0')
|
||||
|
||||
assert(predicate({ run: { dayOfWeek: 0 } }))
|
||||
assert(!predicate({ run: { dayOfWeek: 1 } }))
|
||||
})
|
||||
|
||||
it('chainLength:>5', () => {
|
||||
const predicate = compileExpression('chainLength:>5')
|
||||
|
||||
assert(predicate({ chainLength: 6 }))
|
||||
assert(!predicate({ chainLength: 5 }))
|
||||
})
|
||||
|
||||
it('vm:tags:prod', () => {
|
||||
const predicate = compileExpression('vm:tags:prod')
|
||||
|
||||
assert(predicate({ vm: { tags: ['prod'] } }))
|
||||
assert(predicate({ vm: { tags: ['prod', 'test'] } }))
|
||||
assert(!predicate({ vm: { tags: ['test'] } }))
|
||||
assert(!predicate({ vm: { tags: [] } }))
|
||||
})
|
||||
|
||||
it('run:dayOfWeek:0 chainLength:>3', () => {
|
||||
const predicate = compileExpression('run:dayOfWeek:0 chainLength:>3')
|
||||
|
||||
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
|
||||
assert(!predicate({ run: { dayOfWeek: 0 }, chainLength: 2 }))
|
||||
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
|
||||
})
|
||||
|
||||
it('|(run:dayOfWeek:0 chainLength:>6)', () => {
|
||||
const predicate = compileExpression('|(run:dayOfWeek:0 chainLength:>6)')
|
||||
|
||||
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 7 }))
|
||||
assert(predicate({ run: { dayOfWeek: 1 }, chainLength: 7 }))
|
||||
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
|
||||
assert(!predicate({ run: { dayOfWeek: 1 }, chainLength: 6 }))
|
||||
})
|
||||
|
||||
it('missing field', () => {
|
||||
const predicate = compileExpression('run:dayOfWeek:0')
|
||||
|
||||
assert(predicate({ run: { dayOfWeek: 0 }, chainLength: 6 }))
|
||||
assert(!predicate({ chainLength: 6 }))
|
||||
assert(!predicate({ run: {}, chainLength: 6 }))
|
||||
})
|
||||
|
||||
it('null-checking', () => {
|
||||
const predicate = compileExpression('!vm')
|
||||
|
||||
assert(predicate({}))
|
||||
assert(!predicate({ vm: [{ vm: 'vm' }] }))
|
||||
})
|
||||
|
||||
it('throws on invalid expression', () => {
|
||||
assert.throws(() => compileExpression('vm:('))
|
||||
})
|
||||
})
|
||||
|
||||
describe('isExpression', () => {
|
||||
it('non-empty string', () => {
|
||||
assert(isExpression('vm:tags:test'))
|
||||
})
|
||||
|
||||
it('number', () => {
|
||||
assert(!isExpression(0))
|
||||
})
|
||||
|
||||
it('string[]', () => {
|
||||
assert(!isExpression(['string']))
|
||||
})
|
||||
|
||||
it('undefined', () => {
|
||||
assert(!isExpression(undefined))
|
||||
})
|
||||
|
||||
it('empty string', () => {
|
||||
assert(!isExpression(''))
|
||||
})
|
||||
|
||||
it('number', () => {
|
||||
assert(!isExpression('3'))
|
||||
})
|
||||
|
||||
it('number with spaces', () => {
|
||||
assert(!isExpression(' 5 '))
|
||||
})
|
||||
})
|
||||
@@ -1,7 +1,7 @@
|
||||
import { strict as assert } from 'node:assert'
|
||||
import test from 'node:test'
|
||||
import { execFile } from 'node:child_process'
|
||||
import { copyFile, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
|
||||
import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { Stream } from 'node:stream'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -9,7 +9,6 @@ import { promisify } from 'node:util'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import * as tar from 'tar'
|
||||
|
||||
import { RemoteAdapter } from './RemoteAdapter.mjs'
|
||||
|
||||
@@ -72,83 +71,6 @@ test('fetchPartitionFiles resolves with a node:stream for every format', async t
|
||||
}
|
||||
})
|
||||
|
||||
async function readAll(stream) {
|
||||
const chunks = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
|
||||
async function listTgzEntries(archive, tmp) {
|
||||
const file = join(tmp, 'archive.tgz')
|
||||
await writeFile(file, archive)
|
||||
const entries = []
|
||||
await tar.t({ file, onReadEntry: entry => entries.push(entry.path.replace(/\/$/, '')) })
|
||||
return entries
|
||||
}
|
||||
|
||||
// The partition is mounted on the XOA/proxy: a guest symlink is resolved against the host root,
|
||||
// so following one would expose host files. Paths through a symlink are refused, and symlinks
|
||||
// met while walking a directory are skipped.
|
||||
test('file restore never follows symbolic links', async t => {
|
||||
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-symlink-'))
|
||||
t.after(() => rm(tmp, { recursive: true, force: true }))
|
||||
|
||||
// `host` stands for the XOA filesystem, `root` for the mounted guest partition
|
||||
const host = join(tmp, 'host')
|
||||
const root = join(tmp, 'root')
|
||||
await mkdir(host)
|
||||
await writeFile(join(host, 'secret.txt'), 'host secret')
|
||||
await mkdir(join(root, 'dir'), { recursive: true })
|
||||
await writeFile(join(root, 'dir', 'file.txt'), 'guest file')
|
||||
await symlink(host, join(root, 'absolute')) // like `ln -s / foo` in the guest
|
||||
await symlink('../../host', join(root, 'dir', 'relative'))
|
||||
await symlink('file.txt', join(root, 'dir', 'in-tree'))
|
||||
|
||||
const adapter = new RemoteAdapter({})
|
||||
adapter.getPartition = Disposable.factory(async function* () {
|
||||
yield root
|
||||
})
|
||||
|
||||
const refusedPaths = ['/absolute', '/absolute/secret.txt', '/dir/relative', '/dir/relative/host/secret.txt']
|
||||
|
||||
await t.test('listing refuses a path through a symlink', async () => {
|
||||
for (const path of refusedPaths) {
|
||||
await assert.rejects(adapter.listPartitionFiles('disk', 'partition', path), { code: 'ELINK_NOT_SUPPORTED' }, path)
|
||||
}
|
||||
})
|
||||
|
||||
await t.test('listing skips symlinks', async () => {
|
||||
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/'), { 'dir/': {} })
|
||||
assert.deepEqual(await adapter.listPartitionFiles('disk', 'partition', '/dir'), { 'file.txt': {} })
|
||||
})
|
||||
|
||||
for (const format of ['tgz', 'zip']) {
|
||||
await t.test(`${format}: download refuses a path through a symlink`, async () => {
|
||||
for (const path of refusedPaths) {
|
||||
await assert.rejects(
|
||||
adapter.fetchPartitionFiles('disk', 'partition', ['/dir/file.txt', path], format),
|
||||
{ code: 'ELINK_NOT_SUPPORTED' },
|
||||
path
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
await t.test('tgz: symlinks inside a selected directory are skipped', async () => {
|
||||
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'tgz'))
|
||||
assert.deepEqual((await listTgzEntries(archive, tmp)).sort(), ['dir', 'dir/file.txt'])
|
||||
})
|
||||
|
||||
await t.test('zip: symlinks inside a selected directory are skipped', async () => {
|
||||
const archive = await readAll(await adapter.fetchPartitionFiles('disk', 'partition', ['/dir'], 'zip'))
|
||||
assert.ok(archive.includes('dir/file.txt'))
|
||||
assert.equal(archive.includes('dir/relative'), false)
|
||||
assert.equal(archive.includes('dir/in-tree'), false)
|
||||
})
|
||||
})
|
||||
|
||||
test('LVM file-restore against real losetup/dmsetup/lvm', { skip }, async t => {
|
||||
const tmp = await mkdtemp(join(tmpdir(), 'xo-flr-'))
|
||||
const adapter = new RemoteAdapter({})
|
||||
|
||||
@@ -66,29 +66,7 @@ export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
|
||||
}))
|
||||
|
||||
const makeRelative = path => resolve('/', path).slice(1)
|
||||
|
||||
// The partition is mounted on the XOA/proxy, so the kernel resolves a guest symlink against the
|
||||
// host root: following one (e.g. `ln -s / foo` in the VM, then `/foo/etc`) would read host files.
|
||||
// Symlinks are therefore not supported by file restore: a path going through one is refused,
|
||||
// checked component by component with lstat. The mount is read-only, so no check/use race.
|
||||
export async function resolveWithoutSymlinks(rootPath, path) {
|
||||
const relativePath = makeRelative(path)
|
||||
let currentPath = rootPath
|
||||
for (const name of relativePath.split('/')) {
|
||||
if (name === '') {
|
||||
continue
|
||||
}
|
||||
currentPath = join(currentPath, name)
|
||||
const stats = await lstat(currentPath)
|
||||
if (stats.isSymbolicLink()) {
|
||||
const error = new Error(`symbolic links are not supported by file restore: ${path}`)
|
||||
error.code = 'ELINK_NOT_SUPPORTED'
|
||||
error.path = path
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return relativePath
|
||||
}
|
||||
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
|
||||
|
||||
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
|
||||
for (const relativePath of relativePaths) {
|
||||
@@ -312,12 +290,6 @@ export const fileRestoreMethods = {
|
||||
const self = this
|
||||
Disposable.use(async function* () {
|
||||
const path = yield self.getPartition(diskId, partitionId)
|
||||
// validated before resolving the promise, so a refused path rejects instead of
|
||||
// producing a truncated archive
|
||||
const relativePaths = []
|
||||
for (const requestedPath of paths) {
|
||||
relativePaths.push(await resolveWithoutSymlinks(path, requestedPath))
|
||||
}
|
||||
let outputStream
|
||||
|
||||
if (format === 'tgz') {
|
||||
@@ -326,13 +298,7 @@ export const fileRestoreMethods = {
|
||||
// simultaneous reads. Those saturate the libuv threadpool and starve
|
||||
// the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
|
||||
// threadpool deadlock). Serializing keeps a worker free for them.
|
||||
//
|
||||
// symlinks found while walking the selected directories are skipped, like in the zip
|
||||
// format and the file listing
|
||||
const pack = tar.c(
|
||||
{ cwd: path, gzip: true, jobs: 1, filter: (_, stats) => !stats.isSymbolicLink() },
|
||||
relativePaths
|
||||
)
|
||||
const pack = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
|
||||
|
||||
// node-tar >= 7 returns a Minipass stream: it has `pipe()` but is not a `node:stream`
|
||||
// instance. xo-proxy serves this stream through Koa, which pipes only bodies passing
|
||||
@@ -350,7 +316,7 @@ export const fileRestoreMethods = {
|
||||
// appear as a freeze with no response sent.
|
||||
outputStream = zip.outputStream
|
||||
resolve(zip.outputStream)
|
||||
await addZipEntries(zip, path, '', relativePaths)
|
||||
await addZipEntries(zip, path, '', paths.map(makeRelative))
|
||||
zip.end()
|
||||
} else {
|
||||
throw new Error('unsupported format ' + format)
|
||||
@@ -365,23 +331,17 @@ export const fileRestoreMethods = {
|
||||
},
|
||||
|
||||
async *getDisk(diskId) {
|
||||
try {
|
||||
if (this._useGetDiskLegacy) {
|
||||
yield* getDiskLegacy(this._handler, diskId)
|
||||
return
|
||||
}
|
||||
const handler = this._handler
|
||||
// this is a disposable
|
||||
const mountDir = yield getTmpDir()
|
||||
// this is also a disposable
|
||||
yield mount(handler, diskId, mountDir)
|
||||
// this will yield disk path to caller
|
||||
yield `${mountDir}/vhd0`
|
||||
} finally {
|
||||
// Disposable.factory disposes by calling gen.return(), which runs this finally before
|
||||
// the mount/tmpdir disposers — i.e. exactly as the disk is unmounted.
|
||||
this._partitionsCache.delete(diskId)
|
||||
if (this._useGetDiskLegacy) {
|
||||
yield* getDiskLegacy(this._handler, diskId)
|
||||
return
|
||||
}
|
||||
const handler = this._handler
|
||||
// this is a disposable
|
||||
const mountDir = yield getTmpDir()
|
||||
// this is also a disposable
|
||||
yield mount(handler, diskId, mountDir)
|
||||
// this will yield disk path to caller
|
||||
yield `${mountDir}/vhd0`
|
||||
},
|
||||
|
||||
// partitionId values:
|
||||
@@ -411,7 +371,7 @@ export const fileRestoreMethods = {
|
||||
|
||||
listPartitionFiles(diskId, partitionId, path) {
|
||||
return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
|
||||
path = join(rootPath, await resolveWithoutSymlinks(rootPath, path))
|
||||
path = resolveSubpath(rootPath, path)
|
||||
const entriesMap = {}
|
||||
await asyncEach(
|
||||
await readdir(path),
|
||||
@@ -437,13 +397,7 @@ export const fileRestoreMethods = {
|
||||
},
|
||||
|
||||
listPartitions(diskId) {
|
||||
const cache = this._partitionsCache
|
||||
const cached = cache.get(diskId)
|
||||
if (cached !== undefined) {
|
||||
return cached
|
||||
}
|
||||
|
||||
const pPartitions = Disposable.use(this.getDisk(diskId), async devicePath => {
|
||||
return Disposable.use(this.getDisk(diskId), async devicePath => {
|
||||
// partx may return empty on FUSE-backed files (vhd0); a loop device
|
||||
// presents proper block-device semantics that partx reads reliably.
|
||||
// losetup may itself fail if the FUSE mount isn't fully ready yet —
|
||||
@@ -461,14 +415,11 @@ export const fileRestoreMethods = {
|
||||
}
|
||||
}
|
||||
|
||||
// direct listing failed, try with lvm listing
|
||||
if (partitions.length === 0) {
|
||||
debug('direct listing of the volume partition failed, try to read through LVM')
|
||||
try {
|
||||
// handle potential raw LVM physical volume
|
||||
return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
|
||||
} catch (error) {
|
||||
warn('LVM listing of this volume failed ', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
@@ -504,16 +455,6 @@ export const fileRestoreMethods = {
|
||||
})
|
||||
return results
|
||||
})
|
||||
|
||||
// The entry is evicted on disk unmount (see getDisk)
|
||||
// a failed probe is not cached so the next call retries.
|
||||
cache.set(diskId, pPartitions)
|
||||
pPartitions.catch(() => {
|
||||
if (cache.get(diskId) === pPartitions) {
|
||||
cache.delete(diskId)
|
||||
}
|
||||
})
|
||||
return pPartitions
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -294,9 +294,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
await asyncMap(Object.values(vdiVbds), vbd =>
|
||||
xapi.VBD_create({
|
||||
...vbd,
|
||||
// an uncached live mount serves its disk read only: attach it as such, instead of letting
|
||||
// the guest discover it through I/O errors on its first write
|
||||
mode: isLiveMounted && vdi.liveMountedReadOnly ? 'RO' : vbd.mode,
|
||||
// a live mount serves its disk read only: attach it as such, instead of letting the
|
||||
// guest discover it through I/O errors on its first write
|
||||
mode: isLiveMounted ? 'RO' : vbd.mode,
|
||||
VDI: newVdi.$ref,
|
||||
VM: vmRef,
|
||||
})
|
||||
@@ -307,16 +307,9 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
})
|
||||
|
||||
// 3.5. Destroy old VDIs that are no longer attached to the VM.
|
||||
// Some storage backends refuse to destroy a VDI that still has snapshot children.
|
||||
// Log it and leave it for later cleanup.
|
||||
await asyncMap([...oldVdiRefs], ref =>
|
||||
xapi.call('VDI.destroy', ref).catch(error =>
|
||||
Task.warning('failed to destroy orphaned VDI', {
|
||||
vdi: xapi.getObject(ref, undefined)?.uuid ?? ref,
|
||||
error,
|
||||
})
|
||||
)
|
||||
)
|
||||
// Uses ignoreErrors because some storage backends refuse to destroy a VDI that still has snapshot children;
|
||||
// those VDIs will become truly orphaned once the old snapshot VMs are cleaned up by _deleteOldEntries.
|
||||
await asyncMap([...oldVdiRefs], ref => ignoreErrors.call(xapi.call('VDI.destroy', ref)))
|
||||
|
||||
// 4. For updates, destroy existing VIFs before recreating them.
|
||||
if (isUpdate) {
|
||||
|
||||
@@ -60,7 +60,7 @@ const parsePartxLine = createParser({
|
||||
export async function listPartitions(devicePath) {
|
||||
const parts = await fromCallback(execFile, 'partx', [
|
||||
'--bytes',
|
||||
'--output=START,SIZE,NAME,UUID,TYPE',
|
||||
'--output=NR,START,SIZE,NAME,UUID,TYPE',
|
||||
'--pairs',
|
||||
devicePath,
|
||||
]).catch(error => {
|
||||
|
||||
@@ -10,8 +10,6 @@ import { FullRemote } from './_vmRunners/FullRemote.mjs'
|
||||
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
import { compileExpression, isExpression } from '../_expressionPredicate.mjs'
|
||||
import { buildRunContext, buildVmContext } from '../_buildContext.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -62,25 +60,21 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
return
|
||||
}
|
||||
|
||||
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
|
||||
const predicate = isExpression(settings.vmFilter) ? compileExpression(settings.vmFilter) : null
|
||||
|
||||
const vmsUuids = await sourceRemoteAdapter.listAllVms()
|
||||
|
||||
const queue = new Set(vmsUuids)
|
||||
|
||||
Task.info('vms', { vms: vmsUuids })
|
||||
let nbVms = vmsUuids.length
|
||||
const nbVms = vmsUuids.length
|
||||
let nbVmsDone = 0
|
||||
|
||||
remoteAdapters = getAdaptersByRemote(remoteAdapters)
|
||||
const allSettings = this._job.settings
|
||||
const baseSettings = this._baseSettings
|
||||
|
||||
const queue = new Set(vmsUuids)
|
||||
const taskByVmId = {}
|
||||
const nTriesByVmId = {}
|
||||
|
||||
const handleVm = async vmUuid => {
|
||||
const handleVm = vmUuid => {
|
||||
if (nTriesByVmId[vmUuid] === undefined) {
|
||||
nTriesByVmId[vmUuid] = 0
|
||||
}
|
||||
@@ -90,26 +84,12 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
const vmSettings = { ...settings, ...allSettings[vmUuid] }
|
||||
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
|
||||
|
||||
const vmBackups = await sourceRemoteAdapter.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
|
||||
|
||||
// avoiding to create tasks for empty directories
|
||||
if (vmBackups.length === 0) {
|
||||
nbVms--
|
||||
return
|
||||
}
|
||||
|
||||
if (predicate !== null && !predicate(buildVmContext(vmBackups.at(-1).vm, runContext))) {
|
||||
nbVms--
|
||||
return
|
||||
}
|
||||
|
||||
const opts = {
|
||||
baseSettings,
|
||||
config,
|
||||
job,
|
||||
healthCheckSr,
|
||||
remoteAdapters,
|
||||
runContext,
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
sourceRemoteAdapter,
|
||||
@@ -126,39 +106,45 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
|
||||
}
|
||||
|
||||
if (taskByVmId[vmUuid] === undefined) {
|
||||
taskByVmId[vmUuid] = new Task(taskStart)
|
||||
}
|
||||
const task = taskByVmId[vmUuid]
|
||||
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
|
||||
let taskError
|
||||
return task
|
||||
.runInside(async () =>
|
||||
vmBackup.run().catch(error => {
|
||||
taskError = error
|
||||
})
|
||||
)
|
||||
.then(result => {
|
||||
if (taskError === undefined) {
|
||||
nbVmsDone++
|
||||
return task.success(result)
|
||||
return sourceRemoteAdapter
|
||||
.listVmBackups(vmUuid, ({ mode }) => mode === job.mode)
|
||||
.then(vmBackups => {
|
||||
// avoiding to create tasks for empty directories
|
||||
if (vmBackups.length > 0) {
|
||||
if (taskByVmId[vmUuid] === undefined) {
|
||||
taskByVmId[vmUuid] = new Task(taskStart)
|
||||
}
|
||||
const task = taskByVmId[vmUuid]
|
||||
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
|
||||
let taskError
|
||||
return task
|
||||
.runInside(async () =>
|
||||
vmBackup.run().catch(error => {
|
||||
taskError = error
|
||||
})
|
||||
)
|
||||
.then(result => {
|
||||
if (taskError === undefined) {
|
||||
nbVmsDone++
|
||||
return task.success(result)
|
||||
}
|
||||
if (isLastRun) {
|
||||
nbVmsDone++
|
||||
return task.failure(taskError)
|
||||
}
|
||||
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
|
||||
// don't end the task
|
||||
task.warning(`Retry the VM mirror backup due to an error`, {
|
||||
attempt: nTriesByVmId[vmUuid],
|
||||
error: taskError.message,
|
||||
isRetry: true,
|
||||
})
|
||||
queue.add(vmUuid)
|
||||
})
|
||||
.catch(noop)
|
||||
}
|
||||
if (isLastRun) {
|
||||
nbVmsDone++
|
||||
return task.failure(taskError)
|
||||
}
|
||||
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
|
||||
// don't end the task
|
||||
task.warning(`Retry the VM mirror backup due to an error`, {
|
||||
attempt: nTriesByVmId[vmUuid],
|
||||
error: taskError.message,
|
||||
isRetry: true,
|
||||
})
|
||||
queue.add(vmUuid)
|
||||
})
|
||||
.catch(noop)
|
||||
}
|
||||
|
||||
const { concurrency } = settings
|
||||
const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ import { Throttle } from '@vates/generator-toolbox'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
import { selectSynchronizedSnapshotVms } from './_selectSynchronizedSnapshotVms.mjs'
|
||||
import { buildRunContext } from '../_buildContext.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -69,7 +68,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
const getSnapshotNameLabel = this._getSnapshotNameLabel
|
||||
const schedule = this._schedule
|
||||
const settings = this._settings
|
||||
const runContext = job.runContext ?? buildRunContext(new Date(), settings.timezone)
|
||||
|
||||
const throttleGenerator = new Throttle(settings.maxExportRate)
|
||||
const throttleStream = createStreamThrottle(settings.maxExportRate)
|
||||
@@ -143,6 +141,8 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
await asyncEach(
|
||||
[...vms].filter(vm => batchIds.has(vm.uuid)),
|
||||
async vm => {
|
||||
// The task deliverately includes both the transfer (including the wait) and and backup.
|
||||
// This is done to have the snapshot reported under the VM.
|
||||
const { task } = getVmTask(vm.uuid, vm.name_label)
|
||||
|
||||
const vmSettings = { ...settings, ...allSettings[vm.uuid] }
|
||||
@@ -153,7 +153,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
healthCheckSr,
|
||||
job,
|
||||
remoteAdapters,
|
||||
runContext,
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
srs,
|
||||
@@ -252,7 +251,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
runContext,
|
||||
}
|
||||
|
||||
const vmBackup = this._getVmBackup(job.mode, opts)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { AbstractRemote } from './_AbstractRemote.mjs'
|
||||
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
||||
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
|
||||
@@ -7,8 +5,6 @@ import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
const { warn } = createLogger('xo:backups:FullRemoteVmBackup')
|
||||
|
||||
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
|
||||
_getRemoteWriters() {
|
||||
return [FullRemoteWriter, AggregatedFullRemoteWriter]
|
||||
@@ -25,23 +21,12 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
||||
for (const metadata of transferList) {
|
||||
const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
|
||||
const sizeContainer = watchStreamSize(stream)
|
||||
let checksum
|
||||
try {
|
||||
// the unencrypted destinations store the same data as the source: no need to compute their checksum again
|
||||
checksum = await this._sourceRemoteAdapter.readFullVmBackupChecksum(metadata)
|
||||
} catch (error) {
|
||||
warn('error while copying reading source checksum, recomputing it', error)
|
||||
}
|
||||
|
||||
// a single writer can consume the stream itself: no need to fork it
|
||||
const isForked = this._writers.size > 1
|
||||
|
||||
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
|
||||
await this._callWriters(
|
||||
writer =>
|
||||
writer.run({
|
||||
checksum,
|
||||
stream: isForked ? forkStreamUnpipe(stream) : stream,
|
||||
stream: forkStreamUnpipe(stream),
|
||||
// stream will be forked and transformed, it's not safe to attach additional properties to it
|
||||
streamLength: stream.length,
|
||||
maxStreamLength: stream.maxStreamLength, // for encrypted destination/source without length
|
||||
@@ -53,7 +38,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
||||
'writer.run()'
|
||||
)
|
||||
// for healthcheck
|
||||
this._vm = metadata.vm
|
||||
this._tags = metadata.vm.tags
|
||||
nbTransferredVms++
|
||||
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
|
||||
}
|
||||
|
||||
@@ -62,33 +62,14 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
})
|
||||
// yeah , let's go
|
||||
}
|
||||
|
||||
/**
|
||||
* the block files can be copied as stored, never decompressed nor recompressed, if all the writers write VHD
|
||||
* directories with the same compression (the source disks must also use it, checked per disk when reading)
|
||||
* @returns {string | undefined} the common compression, undefined if a writer writes plain VHD files or if they differ
|
||||
*/
|
||||
#getRawBlocksCompression() {
|
||||
const compressions = new Set([...this._writers].map(writer => writer.getVhdDirectoryCompression()))
|
||||
// a single undefined value means plain VHD files on every writer: no raw blocks either
|
||||
return compressions.size === 1 ? [...compressions][0] : undefined
|
||||
}
|
||||
|
||||
async _run() {
|
||||
const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
|
||||
// the source remote writes with the same compression as all the writers: the blocks can be copied as stored
|
||||
// this is based on the remote configurations, each source disk checks its own compression since it may predate a
|
||||
// configuration change, and falls back to a normal read
|
||||
const writersCompression = this.#getRawBlocksCompression()
|
||||
const rawBlocksCompression =
|
||||
writersCompression === this._sourceRemoteAdapter.getVhdDirectoryCompression() ? writersCompression : undefined
|
||||
const nbTransferrableVms = transferList.length
|
||||
let nbTransferredVms = 0
|
||||
for (const metadata of transferList) {
|
||||
assert.strictEqual(metadata.mode, 'delta')
|
||||
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
|
||||
useChain: false,
|
||||
rawBlocksCompression,
|
||||
})
|
||||
// don't trust metadata too much
|
||||
// recompute if it's a base backup
|
||||
@@ -131,7 +112,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
|
||||
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
|
||||
// for healthcheck
|
||||
this._vm = metadata.vm
|
||||
this._tags = metadata.vm.tags
|
||||
nbTransferredVms++
|
||||
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
|
||||
}
|
||||
|
||||
@@ -18,19 +18,10 @@ import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
|
||||
import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
|
||||
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
|
||||
import { buildVmContext } from '../../_buildContext.mjs'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
|
||||
|
||||
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
|
||||
const fullInterval = this._settings.fullInterval
|
||||
this._fullIntervalPredicate = isExpression(fullInterval) ? compileExpression(fullInterval) : null
|
||||
}
|
||||
|
||||
_getWriters() {
|
||||
return [
|
||||
IncrementalRemoteWriter,
|
||||
@@ -238,17 +229,15 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
}
|
||||
|
||||
// we do tafter checkbasevdis because we want the writer to know the target VM
|
||||
// especially on replication when we always update the same VM
|
||||
// especially on replication when we alwayr update the same VM
|
||||
const fullInterval = this._settings.fullInterval
|
||||
const chainLength = (deltaChainLength ?? 0) + 1
|
||||
if (isExpression(fullInterval)) {
|
||||
const context = buildVmContext(this._vm, this._runContext, chainLength)
|
||||
if (this._fullIntervalPredicate(context)) {
|
||||
debug('forcing full backup: fullInterval expression matched', { fullInterval })
|
||||
return
|
||||
}
|
||||
} else if (fullInterval !== 0 && fullInterval <= chainLength) {
|
||||
debug('not using base VM because fullInterval reached', { fullInterval, chainLength })
|
||||
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
|
||||
debug('not using base VM because fullInterval reached', {
|
||||
fullInterval,
|
||||
deltaChainLength,
|
||||
eq: fullInterval < deltaChainLength + 1,
|
||||
dc1: deltaChainLength + 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { Task } from '@vates/task'
|
||||
import { compileExpression, isExpression } from '../../_expressionPredicate.mjs'
|
||||
import { buildVmContext } from '../../_buildContext.mjs'
|
||||
|
||||
const { debug, warn } = createLogger('xo:backups:AbstractVmRunner')
|
||||
|
||||
@@ -75,58 +73,24 @@ export const Abstract = class AbstractVmBackupRunner {
|
||||
}
|
||||
|
||||
// check if current VM has tags
|
||||
const tags = this._vm.tags
|
||||
const tags = this._tags
|
||||
|
||||
// accept both 'xo:no-health-check' and 'xo:no-health-check=reason'
|
||||
if (tags.some(t => t.startsWith('xo:no-health-check'))) {
|
||||
const vmAlwaysIgnored = tags.some(t => t.startsWith('xo:no-health-check'))
|
||||
const intersect = !vmAlwaysIgnored && settings.healthCheckVmsWithTags.some(t => tags.includes(t))
|
||||
|
||||
if (vmAlwaysIgnored || (settings.healthCheckVmsWithTags.length !== 0 && !intersect)) {
|
||||
// create a task to have an info in the logs and reports
|
||||
return Task.run(
|
||||
{
|
||||
properties: {
|
||||
name: 'health check',
|
||||
},
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
() => {
|
||||
Task.info(`This VM is excluded from health checks via tag`)
|
||||
Task.info(`This VM doesn't match the health check's tags for this schedule`)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
if (isExpression(settings.healthCheckVmsWithTags)) {
|
||||
const context = buildVmContext(this._vm, this._runContext)
|
||||
if (!this._healthCheckPredicate(context)) {
|
||||
return Task.run(
|
||||
{
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
() => {
|
||||
Task.info(`VM did not match healthCheckVmsWithTags expression`)
|
||||
}
|
||||
)
|
||||
}
|
||||
} else {
|
||||
const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
|
||||
if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
|
||||
// create a task to have an info in the logs and reports
|
||||
return Task.run(
|
||||
{
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
() => {
|
||||
Task.info(`This VM doesn't match the health check's tags for this schedule`)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
await this._callWriters(writer => writer.healthCheck(), 'writer.healthCheck()')
|
||||
}
|
||||
|
||||
_compileHealthCheckPredicate() {
|
||||
const { healthCheckVmsWithTags } = this._settings
|
||||
if (isExpression(healthCheckVmsWithTags)) {
|
||||
this._healthCheckPredicate = compileExpression(healthCheckVmsWithTags)
|
||||
} else {
|
||||
this._healthCheckPredicate = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
job,
|
||||
healthCheckSr,
|
||||
remoteAdapters,
|
||||
runContext,
|
||||
schedule,
|
||||
settings,
|
||||
sourceRemoteAdapter,
|
||||
@@ -41,14 +40,11 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
this._throttleGenerator = throttleGenerator
|
||||
this._throttleStream = throttleStream
|
||||
this._vmUuid = vmUuid
|
||||
this._runContext = runContext
|
||||
|
||||
const allSettings = job.settings
|
||||
const writers = new Set()
|
||||
this._writers = writers
|
||||
|
||||
this._compileHealthCheckPredicate()
|
||||
|
||||
const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
|
||||
if (settings.distributeBackups) {
|
||||
writers.add(
|
||||
|
||||
@@ -36,7 +36,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
job,
|
||||
remoteAdapters,
|
||||
remotes,
|
||||
runContext,
|
||||
schedule,
|
||||
settings,
|
||||
srs,
|
||||
@@ -64,13 +63,12 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
this.timestamp = undefined
|
||||
|
||||
// VM currently backed up
|
||||
const tags = vm.tags
|
||||
const tags = (this._tags = vm.tags)
|
||||
|
||||
// VM (snapshot) that is really exported
|
||||
this._exportedVm = undefined
|
||||
this._synchronizedSnapshotTimestamp = synchronizedSnapshotTimestamp
|
||||
this._vm = vm
|
||||
this._runContext = runContext
|
||||
|
||||
this._getSnapshotNameLabel = getSnapshotNameLabel
|
||||
this._isIncremental = job.mode === 'delta'
|
||||
@@ -92,9 +90,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
settings.offlineSnapshot = true
|
||||
}
|
||||
this._settings = settings
|
||||
|
||||
this._compileHealthCheckPredicate()
|
||||
|
||||
// Create writers
|
||||
{
|
||||
const writers = new Set()
|
||||
|
||||
@@ -13,9 +13,7 @@ export class TaskProgressHandler {
|
||||
if (progress < 0 || progress > 1) {
|
||||
return
|
||||
}
|
||||
// the end is always emitted: throttled, a last step under the threshold would leave the task at 99%
|
||||
if (
|
||||
progress !== 1 &&
|
||||
this.#lastProgressDate !== undefined &&
|
||||
this.#lastProgressValue !== undefined &&
|
||||
Date.now() - this.#lastProgressDate < MAX_DURATION_BETWEEN_PROGRESS_EMIT &&
|
||||
|
||||
@@ -1,15 +1,5 @@
|
||||
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
|
||||
export class AggregatedIncrementalRemoteWriter extends AbstractAggregatedRemoteWriter {
|
||||
/**
|
||||
* the backup can land on any of the adapters, the compression is only known if they all use the same one
|
||||
* (based on the adapters since the child writers are only created in beforeBackup())
|
||||
* @returns {string | undefined} see RemoteAdapter#getVhdDirectoryCompression
|
||||
*/
|
||||
getVhdDirectoryCompression() {
|
||||
const compressions = new Set(Object.values(this.adapters).map(adapter => adapter.getVhdDirectoryCompression()))
|
||||
return compressions.size === 1 ? [...compressions][0] : undefined
|
||||
}
|
||||
|
||||
async deleteOldBackupsOnAdapter(adapter, backups) {
|
||||
await adapter.deleteDeltaVmBackups(backups)
|
||||
}
|
||||
|
||||
@@ -6,14 +6,13 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
|
||||
/**
|
||||
*
|
||||
* @param {Map<string,string>} baseUuidToSrcVdi
|
||||
* @param {Map<string,string>} contentKeys
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi, contentKeys) {
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
debug('checkBaseVdis', { baseUuidToSrcVdi })
|
||||
let selectedCopy = new Map()
|
||||
for (const writer of this.writers) {
|
||||
const copy = new Map(baseUuidToSrcVdi)
|
||||
await writer.checkBaseVdis(copy, contentKeys)
|
||||
await writer.checkBaseVdis(copy)
|
||||
if (copy.size > 0) {
|
||||
debug('checkBaseVdis found a mainwriter candidate ', writer._sr.name_label)
|
||||
// there can be multiple candidates
|
||||
@@ -35,17 +34,17 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
this.setOldReplicaList()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
}
|
||||
|
||||
async prepare(args) {
|
||||
const { settings } = this.props
|
||||
if (settings.deleteFirst) {
|
||||
await this.deleteOldReplicas({ keepMostRecent: !args.isFull && settings.copyRetention === 1 })
|
||||
}
|
||||
if (this.mainWriter === undefined) {
|
||||
debug('no mainwriter found, fallback to a new SR')
|
||||
// use the SR with the most empty space for starting a new chain
|
||||
@@ -63,7 +62,9 @@ export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWrite
|
||||
// remove the backups and remove the entries
|
||||
async cleanup() {
|
||||
debug('cleanup')
|
||||
await this.deleteOldReplicas()
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.cleanup()))
|
||||
}
|
||||
|
||||
|
||||
@@ -25,11 +25,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} params
|
||||
* @param {string} [params.checksum] - checksum of the stream data if known (full mirror of an unencrypted backup)
|
||||
*/
|
||||
async _run({ checksum, maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
|
||||
async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
|
||||
const settings = this._settings
|
||||
const job = this._job
|
||||
const scheduleId = this._schedule.id
|
||||
@@ -71,7 +67,6 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
|
||||
await Task.run({ properties: { name: 'transfer' } }, async () => {
|
||||
await adapter.outputStream(dataFilename, stream, {
|
||||
checksum,
|
||||
maxStreamLength,
|
||||
streamLength,
|
||||
validator: tmpPath => adapter.isValidXva(tmpPath),
|
||||
|
||||
@@ -25,14 +25,6 @@ const { warn } = createLogger('xo:backups:DeltaBackupWriter')
|
||||
export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
|
||||
#parentVdiPaths
|
||||
#parentUuids
|
||||
|
||||
/**
|
||||
* @returns {string | undefined} see RemoteAdapter#getVhdDirectoryCompression
|
||||
*/
|
||||
getVhdDirectoryCompression() {
|
||||
return this._adapter.getVhdDirectoryCompression()
|
||||
}
|
||||
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
this.#parentVdiPaths = {}
|
||||
this.#parentUuids = {}
|
||||
|
||||
@@ -65,11 +65,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
|
||||
const snapshotCandidates = new Map()
|
||||
|
||||
// skip orphan snapshots (active VDI destroyed): they can't be validated against an active disk
|
||||
const replicaSnapshots = sr.$VDIs.filter(
|
||||
vdi => vdi?.managed && vdi?.is_a_snapshot && vdi.$snapshot_of !== undefined
|
||||
)
|
||||
|
||||
for (const [baseUuid, srcVdiuid] of baseUuidToSrcVdi) {
|
||||
let target
|
||||
|
||||
@@ -77,9 +72,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
|
||||
if (contentKey !== undefined) {
|
||||
debug('got one content key, look for a candidate')
|
||||
target = replicaSnapshots.find(
|
||||
target = sr.$VDIs.find(
|
||||
vdi =>
|
||||
vdi.other_config[CONTENT_KEY] === contentKey &&
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[CONTENT_KEY] === contentKey && // &&
|
||||
// ensure we don't replicate on ourself or any of the vdi in the source chain
|
||||
vdi.$snapshot_of.uuid !== srcVdiuid
|
||||
)
|
||||
@@ -88,11 +85,13 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
// fall back for older snapshots
|
||||
if (target === undefined) {
|
||||
debug('content key not here or not found , look by jobid ')
|
||||
target = replicaSnapshots.find(
|
||||
target = sr.$VDIs.find(
|
||||
vdi =>
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[JOB_ID] === this._job.id &&
|
||||
vdi.other_config[VM_UUID] === this._vmUuid &&
|
||||
vdi.other_config[COPY_OF] === baseUuid
|
||||
vdi?.other_config[COPY_OF] === baseUuid
|
||||
)
|
||||
}
|
||||
if (target !== undefined) {
|
||||
@@ -384,15 +383,14 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
retentionEntries.sort(compareReplicatedVmDatetime)
|
||||
this._oldEntries = getOldEntries(settings.copyRetention - 1, retentionEntries)
|
||||
|
||||
if (settings.deleteFirst && !settings.skipDeleteOldEntries) {
|
||||
if (settings.deleteFirst && settings.skipDeleteOldEntries) {
|
||||
// we want to keep the baseVM when copying a delta
|
||||
// even if we want to keep only one after
|
||||
let mostRecentEntry
|
||||
if (this._oldEntries.length > 0 && settings.copyRetention === 1 && !isFull) {
|
||||
if (this._oldEntries.length > 1 && settings.copyRetention === 1 && !isFull) {
|
||||
mostRecentEntry = this._oldEntries.pop()
|
||||
}
|
||||
await this._deleteOldEntries()
|
||||
await this._deleteOrphanVdis()
|
||||
this._oldEntries = mostRecentEntry !== undefined ? [mostRecentEntry] : []
|
||||
}
|
||||
}
|
||||
@@ -400,7 +398,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
async cleanup() {
|
||||
if (!this._settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
await this._deleteOrphanVdis()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -408,35 +405,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
return asyncMapSettled(this._oldEntries, vm => vm.$destroy({ bypassBlockedOperation: true }))
|
||||
}
|
||||
|
||||
// VDIs from this job/VM/SR that ended up detached from any VM, because a previous run
|
||||
// was interrupted or a storage backend refused to destroy a VDI at import time.
|
||||
// Destroying a VM only reaches the VDIs still attached to it, _deleteOldEntries never cleans them.
|
||||
async _deleteOrphanVdis() {
|
||||
const sr = this._sr
|
||||
const vmUuid = this._vmUuid
|
||||
const jobId = this._job.id
|
||||
|
||||
const orphanVdis = sr.$VDIs.filter(
|
||||
vdi =>
|
||||
vdi?.managed &&
|
||||
vdi.other_config[JOB_ID] === jobId &&
|
||||
vdi.other_config[VM_UUID] === vmUuid &&
|
||||
vdi.other_config[REPLICATED_TO_SR_UUID] === sr.uuid &&
|
||||
vdi.type !== 'suspend' &&
|
||||
// a VBD or VM missing from the cache (e.g. just destroyed) keeps the VDI, next run will handle it
|
||||
vdi.$VBDs.every(vbd => vbd?.$VM?.is_control_domain === true)
|
||||
)
|
||||
|
||||
await asyncEach(
|
||||
orphanVdis,
|
||||
vdi =>
|
||||
sr.$xapi
|
||||
.call('VDI.destroy', vdi.$ref)
|
||||
.catch(error => Task.warning('failed to delete orphan VDI', { vdi: vdi.uuid, error })),
|
||||
{ concurrency: 5 }
|
||||
)
|
||||
}
|
||||
|
||||
#decorateVmMetadata(backup, timestamp) {
|
||||
const { _warmMigration } = this._settings
|
||||
const sr = this._sr
|
||||
|
||||
@@ -61,7 +61,7 @@ export class AbstractAggregatedXapiWriter {
|
||||
|
||||
setOldReplicaList() {
|
||||
debug(`setOldReplicaList for vm ${this.#props.vmUuid}`)
|
||||
const scheduleId = this.#props.schedule.id
|
||||
const scheduleId = this.#props.scheduleId
|
||||
const vmUuid = this.#props.vmUuid
|
||||
const settings = this.#props.settings
|
||||
|
||||
@@ -80,35 +80,14 @@ export class AbstractAggregatedXapiWriter {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the old replicas computed by `setOldReplicaList`.
|
||||
*
|
||||
* @param {Object} [options]
|
||||
* @param {boolean} [options.keepMostRecent=false] - Keep the most recent old replica, which is the base of the
|
||||
* next incremental replication (needed when copyRetention is 1 with deleteFirst). The kept replica stays in
|
||||
* the list, so a later call without `keepMostRecent` (from `cleanup()`) deletes it once the transfer is done.
|
||||
*/
|
||||
async deleteOldReplicas({ keepMostRecent = false } = {}) {
|
||||
let toDelete = this.#oldVmReplicaList
|
||||
let kept
|
||||
|
||||
// the list is sorted by datetime (oldest first), so the last entry is the most recent replica,
|
||||
// which is the base of the next incremental replication
|
||||
if (keepMostRecent && toDelete.length > 0) {
|
||||
kept = toDelete[toDelete.length - 1]
|
||||
toDelete = toDelete.slice(0, -1)
|
||||
}
|
||||
|
||||
debug(`deleteOldReplicas will delete ${toDelete.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
async deleteOldReplicas() {
|
||||
debug(`deleteOldReplicas will delete ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
// destroy the VM on the right xapi
|
||||
await asyncMapSettled(toDelete, async vm => {
|
||||
await asyncMapSettled(this.#oldVmReplicaList, async vm => {
|
||||
debug('will delete old replica', vm.name_label)
|
||||
await vm.$xapi.VM_destroy(vm.$ref)
|
||||
})
|
||||
debug(`deleteOldReplicas deleted ${toDelete.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
|
||||
// whatever is left (the kept base, if any) will be deleted by the next call, from cleanup()
|
||||
this.#oldVmReplicaList = kept !== undefined ? [kept] : []
|
||||
debug(`deleteOldReplicas deleted ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
}
|
||||
|
||||
async setupWriters() {
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
|
||||
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
|
||||
import { DATETIME, JOB_ID, REPLICATED_TO_SR_UUID, SCHEDULE_ID, VM_UUID } from '../../_otherConfig.mjs'
|
||||
|
||||
// This exercises setOldReplicaList()/deleteOldReplicas() through the REAL
|
||||
// listReplicatedVms() (same fixture style as ./_listReplicatedVms.test.mjs), rather
|
||||
// than mocking the module. This is deliberate: `node:test`'s module mocking needs
|
||||
// --experimental-test-module-mocks, which this repo's `test` script (and therefore
|
||||
// the pre-commit hook) does not pass, so a mocked version of this test would never
|
||||
// pass CI. Going through the real filtering logic also means this test genuinely
|
||||
// proves the fix — not just that a particular value gets forwarded to a spy.
|
||||
|
||||
const SOURCE_VM_UUID = 'vm-1'
|
||||
const SR_UUID = 'sr-1'
|
||||
const SCHEDULE_A = 'schedule-a'
|
||||
const SCHEDULE_B = 'schedule-b'
|
||||
|
||||
const replicaTarget = { uuid: 'replica-target-uuid' } // stand-in for $snapshot_of on new-style replicas
|
||||
|
||||
/**
|
||||
* Builds a fake "new-style" replication snapshot: a snapshot of the replicated
|
||||
* target VM (not of the source VM), tagged with the schedule/job that produced it.
|
||||
*/
|
||||
const makeReplicaSnapshot = ({ id, datetime, scheduleId, srUuid = SR_UUID, vmUuid = SOURCE_VM_UUID }) => ({
|
||||
$id: id,
|
||||
$type: 'VM',
|
||||
blocked_operations: {},
|
||||
is_a_snapshot: true,
|
||||
is_a_template: false,
|
||||
name_label: id,
|
||||
$snapshot_of: replicaTarget,
|
||||
other_config: {
|
||||
[DATETIME]: datetime,
|
||||
[JOB_ID]: 'job-1',
|
||||
[SCHEDULE_ID]: scheduleId,
|
||||
[VM_UUID]: vmUuid,
|
||||
[REPLICATED_TO_SR_UUID]: srUuid,
|
||||
},
|
||||
})
|
||||
|
||||
const makeXapi = vms => ({ objects: { indexes: { type: { VM: Object.fromEntries(vms.map(vm => [vm.$id, vm])) } } } })
|
||||
|
||||
/** Attaches $xapi/$ref (as deleteOldReplicas expects) and a VM_destroy spy to each vm. */
|
||||
function wireForDestroy(vms, destroyCalls) {
|
||||
const xapi = makeXapi(vms)
|
||||
for (const vm of vms) {
|
||||
vm.$xapi = { objects: xapi.objects, VM_destroy: ref => destroyCalls.push(ref) }
|
||||
vm.$ref = vm.$id
|
||||
}
|
||||
return xapi
|
||||
}
|
||||
|
||||
function makeSr(xapi, uuid = SR_UUID) {
|
||||
return {
|
||||
uuid,
|
||||
name_label: `sr-${uuid}`,
|
||||
physical_utilisation: 0,
|
||||
physical_size: 100,
|
||||
$xapi: xapi,
|
||||
}
|
||||
}
|
||||
|
||||
function makeWriter({ srs, vmUuid = SOURCE_VM_UUID, scheduleId, copyRetention }) {
|
||||
return new AbstractAggregatedXapiWriter({
|
||||
srs,
|
||||
ReplicationWriter: class {},
|
||||
vmUuid,
|
||||
schedule: { id: scheduleId }, // <-- the real shape passed by the caller
|
||||
settings: { copyRetention },
|
||||
job: { settings: {} },
|
||||
})
|
||||
}
|
||||
|
||||
describe('AbstractAggregatedXapiWriter#setOldReplicaList / deleteOldReplicas', () => {
|
||||
it('finds and prunes replicas for its own schedule down to copyRetention', async () => {
|
||||
const replicas = [
|
||||
makeReplicaSnapshot({ id: 'a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A }),
|
||||
makeReplicaSnapshot({ id: 'a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A }),
|
||||
makeReplicaSnapshot({ id: 'a-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A }),
|
||||
makeReplicaSnapshot({ id: 'a-4', datetime: '20260104T000000Z', scheduleId: SCHEDULE_A }),
|
||||
]
|
||||
const destroyCalls = []
|
||||
const xapi = wireForDestroy(replicas, destroyCalls)
|
||||
const writer = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_A, copyRetention: 2 })
|
||||
|
||||
writer.setOldReplicaList()
|
||||
await writer.deleteOldReplicas()
|
||||
|
||||
// copyRetention 2 keeps the running copy + 1 old one => 4 - 1 = 3 old ones pruned.
|
||||
// (If setOldReplicaList still reads props.scheduleId instead of props.schedule.id,
|
||||
// scheduleId is undefined, listReplicatedVms matches nothing, and destroyCalls
|
||||
// stays empty — this assertion catches exactly that regression.)
|
||||
assert.equal(
|
||||
destroyCalls.length,
|
||||
replicas.length - (2 - 1),
|
||||
`expected ${replicas.length - 1} replicas pruned for retention 2, got ${destroyCalls.length}. ` +
|
||||
'An empty destroyCalls list means listReplicatedVms was called with an undefined schedule id ' +
|
||||
'and matched nothing (the scheduleId-vs-schedule.id bug).'
|
||||
)
|
||||
})
|
||||
|
||||
it('does not touch another schedule’s replicas on the same VM/SR when pruning its own', async () => {
|
||||
const scheduleAReplicas = [
|
||||
makeReplicaSnapshot({ id: 'a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A }),
|
||||
makeReplicaSnapshot({ id: 'a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A }),
|
||||
makeReplicaSnapshot({ id: 'a-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A }),
|
||||
]
|
||||
const scheduleBReplicas = [
|
||||
makeReplicaSnapshot({ id: 'b-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_B }),
|
||||
makeReplicaSnapshot({ id: 'b-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_B }),
|
||||
makeReplicaSnapshot({ id: 'b-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_B }),
|
||||
makeReplicaSnapshot({ id: 'b-4', datetime: '20260104T000000Z', scheduleId: SCHEDULE_B }),
|
||||
makeReplicaSnapshot({ id: 'b-5', datetime: '20260105T000000Z', scheduleId: SCHEDULE_B }),
|
||||
]
|
||||
|
||||
// Both schedules' replicas coexist in the same xapi, same SR, same source VM —
|
||||
// exactly the real-world "multiple schedules on one job" case the bug breaks.
|
||||
const allReplicas = [...scheduleAReplicas, ...scheduleBReplicas]
|
||||
const destroyCalls = []
|
||||
const xapi = wireForDestroy(allReplicas, destroyCalls)
|
||||
|
||||
// Prune schedule A only (retention 1: keep the running copy, drop the rest).
|
||||
const writerA = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_A, copyRetention: 1 })
|
||||
writerA.setOldReplicaList()
|
||||
await writerA.deleteOldReplicas()
|
||||
|
||||
const destroyedIds = destroyCalls.slice().sort()
|
||||
assert.deepEqual(
|
||||
destroyedIds,
|
||||
['a-1', 'a-2', 'a-3'],
|
||||
'pruning schedule A should only ever destroy schedule A replicas, never schedule B’s — ' +
|
||||
`got ${JSON.stringify(destroyedIds)}`
|
||||
)
|
||||
|
||||
// Now prune schedule B (retention 2: keep the running copy + 1 old one => drop 4 of 5).
|
||||
const writerB = makeWriter({ srs: [makeSr(xapi)], scheduleId: SCHEDULE_B, copyRetention: 2 })
|
||||
writerB.setOldReplicaList()
|
||||
await writerB.deleteOldReplicas()
|
||||
|
||||
const destroyedAfterB = destroyCalls.slice(3).sort() // calls added since writerA's pass
|
||||
assert.equal(destroyedAfterB.length, scheduleBReplicas.length - (2 - 1))
|
||||
assert.ok(
|
||||
destroyedAfterB.every(id => id.startsWith('b-')),
|
||||
`pruning schedule B should only destroy schedule B replicas — got ${JSON.stringify(destroyedAfterB)}`
|
||||
)
|
||||
})
|
||||
|
||||
it('calls listReplicatedVms scoped to each storage repository in turn', async () => {
|
||||
const srAReplicas = [
|
||||
makeReplicaSnapshot({ id: 'sr-a-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-a' }),
|
||||
makeReplicaSnapshot({ id: 'sr-a-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-a' }),
|
||||
]
|
||||
const srBReplicas = [
|
||||
makeReplicaSnapshot({ id: 'sr-b-1', datetime: '20260101T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
|
||||
makeReplicaSnapshot({ id: 'sr-b-2', datetime: '20260102T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
|
||||
makeReplicaSnapshot({ id: 'sr-b-3', datetime: '20260103T000000Z', scheduleId: SCHEDULE_A, srUuid: 'sr-b' }),
|
||||
]
|
||||
|
||||
const destroyCalls = []
|
||||
// Both SRs happen to share the same underlying xapi object index in this fixture
|
||||
// (as they would if source and destination are on the same pool); what matters is
|
||||
// that each SR's uuid scopes its own replicas.
|
||||
const xapi = wireForDestroy([...srAReplicas, ...srBReplicas], destroyCalls)
|
||||
|
||||
const writer = makeWriter({
|
||||
srs: [makeSr(xapi, 'sr-a'), makeSr(xapi, 'sr-b')],
|
||||
scheduleId: SCHEDULE_A,
|
||||
copyRetention: 1, // keep only the running copy on each SR
|
||||
})
|
||||
|
||||
writer.setOldReplicaList()
|
||||
await writer.deleteOldReplicas()
|
||||
|
||||
assert.deepEqual(
|
||||
destroyCalls.slice().sort(),
|
||||
['sr-a-1', 'sr-a-2', 'sr-b-1', 'sr-b-2', 'sr-b-3'],
|
||||
'retention 1 should prune every replica on every configured SR down to none kept aside'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,18 +1,9 @@
|
||||
import { AbstractWriter } from './_AbstractWriter.mjs'
|
||||
|
||||
export class AbstractFullWriter extends AbstractWriter {
|
||||
async run({ checksum, maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
|
||||
async run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
|
||||
try {
|
||||
return await this._run({
|
||||
checksum,
|
||||
maxStreamLength,
|
||||
timestamp,
|
||||
sizeContainer,
|
||||
stream,
|
||||
streamLength,
|
||||
vm,
|
||||
vmSnapshot,
|
||||
})
|
||||
return await this._run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot })
|
||||
} finally {
|
||||
// ensure stream is properly closed
|
||||
stream.destroy()
|
||||
|
||||
@@ -12,14 +12,11 @@ const { warn } = createLogger('xo:backups:vdiRestoreTargets')
|
||||
*/
|
||||
|
||||
/**
|
||||
* Attach the backup disk itself to the restored VM, through a live mount: no copy, read only —
|
||||
* unless it is cached, which makes it read/write.
|
||||
* Attach the backup disk itself to the restored VM, through a live mount: no copy, read only.
|
||||
*
|
||||
* @typedef {object} LiveMountVdiTarget
|
||||
* @property {'live-mount'} type
|
||||
* @property {string} host - uuid of the host the disk is attached to
|
||||
* @property {string} [cacheSr] - uuid of the SR of the local VDI the disk is materialized into, which
|
||||
* also holds the writes, lost on unmount
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -136,20 +133,6 @@ export function hasLiveMountTarget(mapVdisSrs) {
|
||||
return Object.values(mapVdisSrs ?? {}).some(value => value?.type === 'live-mount')
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a raw `mapVdisSrs` setting asks for at least one cached, hence read/write, live mount.
|
||||
*
|
||||
* Such a mount needs the VM of the appliance serving it, which the caller has to look up. A `null`
|
||||
* cache SR does not ask for one: it is invalid, and left to {@link normalizeVdiRestoreTargets} to
|
||||
* reject, rather than failing on the lookup of a VM which was never needed.
|
||||
*
|
||||
* @param {object} [mapVdisSrs]
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function hasCachedLiveMountTarget(mapVdisSrs) {
|
||||
return Object.values(mapVdisSrs ?? {}).some(value => value?.type === 'live-mount' && value.cacheSr != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize the `mapVdisSrs` restore setting into per-VDI targets.
|
||||
*
|
||||
@@ -203,17 +186,10 @@ function normalizeTarget(vdiUuid, value, useDifferentialRestore) {
|
||||
}
|
||||
|
||||
if (type === 'live-mount') {
|
||||
const { cacheSr, host } = value
|
||||
if (!isNonEmptyString(host)) {
|
||||
if (!isNonEmptyString(value.host)) {
|
||||
throw invalidTarget(vdiUuid, 'a live mount requires the id of the host the disk is attached to')
|
||||
}
|
||||
if (cacheSr === undefined) {
|
||||
return { type: 'live-mount', host }
|
||||
}
|
||||
if (!isNonEmptyString(cacheSr)) {
|
||||
throw invalidTarget(vdiUuid, `expected the uuid of the cache SR, got ${typeof cacheSr}`)
|
||||
}
|
||||
return { type: 'live-mount', host, cacheSr }
|
||||
return { type: 'live-mount', host: value.host }
|
||||
}
|
||||
|
||||
if (type === 'restore') {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, it } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { hasCachedLiveMountTarget, hasLiveMountTarget, normalizeVdiRestoreTargets } from './_vdiRestoreTargets.mjs'
|
||||
import { hasLiveMountTarget, normalizeVdiRestoreTargets } from './_vdiRestoreTargets.mjs'
|
||||
|
||||
const VDI_1 = 'vdi-uuid-1'
|
||||
const VDI_2 = 'vdi-uuid-2'
|
||||
@@ -64,11 +64,6 @@ describe('normalizeVdiRestoreTargets()', () => {
|
||||
assert.equal(targets.getLiveMountHost(), HOST)
|
||||
})
|
||||
|
||||
it('keeps the cache SR of a live mount', () => {
|
||||
const targets = normalizeVdiRestoreTargets({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: SR } })
|
||||
assert.deepEqual(targets.get(VDI_1), { type: 'live-mount', host: HOST, cacheSr: SR })
|
||||
})
|
||||
|
||||
it('accepts a restore without SR', () => {
|
||||
const targets = normalizeVdiRestoreTargets({ [VDI_1]: { type: 'restore' } })
|
||||
assert.deepEqual(targets.get(VDI_1), { type: 'restore', sr: undefined, useDifferential: false })
|
||||
@@ -102,13 +97,6 @@ describe('normalizeVdiRestoreTargets()', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects a non string cache SR', () => {
|
||||
assert.throws(
|
||||
() => normalizeVdiRestoreTargets({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: null } }),
|
||||
/expected the uuid of the cache SR/
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects a non string SR', () => {
|
||||
assert.throws(() => normalizeVdiRestoreTargets({ [VDI_1]: { type: 'restore', sr: 42 } }), /expected an SR uuid/)
|
||||
})
|
||||
@@ -142,20 +130,3 @@ describe('hasLiveMountTarget()', () => {
|
||||
assert.equal(hasLiveMountTarget({ [VDI_1]: SR, [VDI_2]: null, [VDI_3]: { type: 'ignore' } }), false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('hasCachedLiveMountTarget()', () => {
|
||||
it('detects a cached live mount entry', () => {
|
||||
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: SR } }), true)
|
||||
})
|
||||
|
||||
it('is false for an uncached live mount, and every other shape', () => {
|
||||
assert.equal(hasCachedLiveMountTarget(undefined), false)
|
||||
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST } }), false)
|
||||
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: SR, [VDI_2]: null, [VDI_3]: { type: 'ignore' } }), false)
|
||||
})
|
||||
|
||||
// the normalization rejects it, which is the error worth reporting
|
||||
it('is false for a null cache SR', () => {
|
||||
assert.equal(hasCachedLiveMountTarget({ [VDI_1]: { type: 'live-mount', host: HOST, cacheSr: null } }), false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
import { isExpression, compileExpression } from './_expressionPredicate.mjs'
|
||||
import { invalidParameters } from 'xo-common/api-errors.js'
|
||||
|
||||
export function assertValidSettingsExpression(settings, keys) {
|
||||
for (const [scheduleId, scheduleSettings] of Object.entries(settings ?? {})) {
|
||||
for (const key of keys) {
|
||||
assertValidExpression(`settings/${scheduleId === '' ? 'general' : scheduleId}/${key}`, scheduleSettings?.[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function assertValidExpression(name, value) {
|
||||
if (isExpression(value)) {
|
||||
try {
|
||||
compileExpression(value)
|
||||
} catch (error) {
|
||||
throw invalidParameters(`Invalid ${name} expression: ${error.message}`, [
|
||||
{ instancePath: `/${name}`, message: error.message },
|
||||
])
|
||||
}
|
||||
} else if (typeof value === 'string' && value.trim() === '') {
|
||||
throw invalidParameters(`Invalid ${name} expression: must be a non-empty expression`, [
|
||||
{ instancePath: `/${name}`, message: 'must be a non-empty expression' },
|
||||
])
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
import { invalidParameters } from 'xo-common/api-errors.js'
|
||||
|
||||
import { assertValidExpression, assertValidSettingsExpression } from './backupExpressions.mjs'
|
||||
|
||||
const VALID = 'vm:tags:prod'
|
||||
const INVALID = 'vm:('
|
||||
|
||||
// checks that the error is `invalidParameters` and points to the expected field
|
||||
const isInvalidAt = instancePath => error =>
|
||||
invalidParameters.is(error) && error.data.errors[0].instancePath === instancePath
|
||||
|
||||
describe('assertValidExpression', function () {
|
||||
it('accepts a valid expression', function () {
|
||||
assert.doesNotThrow(() => assertValidExpression('srs', VALID))
|
||||
})
|
||||
|
||||
it('rejects an invalid expression with invalidParameters', function () {
|
||||
assert.throws(() => assertValidExpression('srs', INVALID), isInvalidAt('/srs'))
|
||||
})
|
||||
|
||||
it('ignores values which are not expressions', function () {
|
||||
for (const value of [undefined, '3', { id: 'sr-1' }, { id: { __or: ['sr-1', 'sr-2'] } }]) {
|
||||
assert.doesNotThrow(() => assertValidExpression('srs', value))
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects blank strings', function () {
|
||||
for (const value of ['', ' ']) {
|
||||
assert.throws(() => assertValidExpression('srs', value), isInvalidAt('/srs'))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertValidSettingsExpression', function () {
|
||||
const KEYS = ['healthCheckVmsWithTags', 'fullInterval']
|
||||
|
||||
it('accepts valid expressions in the general and the schedule settings', function () {
|
||||
assert.doesNotThrow(() =>
|
||||
assertValidSettingsExpression(
|
||||
{ '': { fullInterval: VALID }, 'schedule-1': { healthCheckVmsWithTags: VALID } },
|
||||
KEYS
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an invalid expression in the schedule settings and points to it', function () {
|
||||
assert.throws(
|
||||
() => assertValidSettingsExpression({ '': {}, 'schedule-1': { fullInterval: INVALID } }, KEYS),
|
||||
isInvalidAt('/settings/schedule-1/fullInterval')
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an invalid expression in the general settings', function () {
|
||||
// the general settings are keyed by '', named "general" in the path.
|
||||
assert.throws(
|
||||
() => assertValidSettingsExpression({ '': { healthCheckVmsWithTags: INVALID } }, KEYS),
|
||||
isInvalidAt('/settings/general/healthCheckVmsWithTags')
|
||||
)
|
||||
})
|
||||
|
||||
it('ignores the legacy array and number values', function () {
|
||||
assert.doesNotThrow(() =>
|
||||
assertValidSettingsExpression({ 'schedule-1': { healthCheckVmsWithTags: ['tag'], fullInterval: 3 } }, KEYS)
|
||||
)
|
||||
})
|
||||
|
||||
it('ignores the keys it has not been asked to check', function () {
|
||||
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': { vmFilter: INVALID } }, KEYS))
|
||||
})
|
||||
|
||||
it('handles missing settings and undefined entries', function () {
|
||||
assert.doesNotThrow(() => assertValidSettingsExpression(undefined, KEYS))
|
||||
assert.doesNotThrow(() => assertValidSettingsExpression({ 'schedule-1': undefined }, KEYS))
|
||||
})
|
||||
})
|
||||
@@ -43,7 +43,7 @@ A VM directory holds **both** the full and the incremental backups of that VM: t
|
||||
|
||||
Transient entries, not listed in the trees below:
|
||||
|
||||
- `xo-vm-backups/<VM UUID>.lock`, held while a job writes to that VM's folder and while `cleanVm` cleans or merges it (after a job, after a deletion, or in the merge worker on local remotes). It is effective on S3 (an object rewritten every 30 seconds) and Azure (blob lease) too. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
|
||||
- `xo-vm-backups/<VM UUID>.lock`, held for the duration of a job run on that VM. When listing VMs, entries starting with `.` and entries ending with `.lock` are ignored.
|
||||
- `xo-vm-backups/.queue/clean-vm/<YYYYMMDD>T<HHmmss>-<random>`, the merge worker queue
|
||||
- `.<VHD file name>.merge.json`, in a VDI directory, interrupted merge states.
|
||||
|
||||
@@ -167,7 +167,6 @@ Append-only log of what happened to the VM backup metadata.
|
||||
- entries are never modified nor overwritten, so it works on an immutable remote
|
||||
- written unencrypted, so it stays readable without the encryption key
|
||||
- one directory per day: a reader lists only the days it misses, a purge drops whole directories
|
||||
- xo-server removes the days older than `backups.journalRetention` from every enabled remote when it starts, then once a day; it always keeps the `fullListingInterval` of the remote, during which it reads the journal back; a day which cannot be removed is skipped, and the next purge tries it again
|
||||
- the time part has a fixed width, so sorting the names by name sorts them by date
|
||||
- `event` is `add`, `change` or `del`, and the file content also has a `reason` field
|
||||
- the name of the file is only there to be read by a human: XO reads every field from the file content
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
|
||||
/**
|
||||
* XAPI names of what a live mount of a backed up disk shows the user, so the ones created by the REST
|
||||
* API and by a restore read the same.
|
||||
*
|
||||
* The VDI attached to the host keeps the name of the backed up disk: that is the disk of the VM as
|
||||
* the user knows it. The SR says what it is, and which restore point it serves, the same way as a
|
||||
* restored VM is named.
|
||||
*
|
||||
* @param {object} params
|
||||
* @param {string} params.vmNameLabel - name of the backed up VM
|
||||
* @param {string} params.vdiNameLabel - name of the backed up disk
|
||||
* @param {number} params.timestamp - of the restore point, in ms
|
||||
* @param {boolean} [params.readWrite] - whether the mount is a live restore rather than a live mount
|
||||
* @returns {{ srNameLabel: string, vdiNameLabel: string, vdiNameDescription: string }} - the `xapiLabels` of the live mount mixin
|
||||
*/
|
||||
export function liveMountXapiLabels({ vmNameLabel, vdiNameLabel, timestamp, readWrite = false }) {
|
||||
const what = readWrite ? 'live restore' : 'live mount'
|
||||
const restorePoint = `${vmNameLabel} (${formatFilenameDate(new Date(timestamp))})`
|
||||
return {
|
||||
srNameLabel: `[XO ${what}] ${restorePoint}`,
|
||||
vdiNameLabel,
|
||||
vdiNameDescription: readWrite
|
||||
? `Live restore of ${restorePoint}: what is written to it is lost on unmount.`
|
||||
: `Read-only live mount of ${restorePoint}, removed on unmount.`,
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
|
||||
import { liveMountXapiLabels } from './liveMountXapiLabels.mjs'
|
||||
|
||||
const TIMESTAMP = 1754035712000 // 2025-08-01T08:08:32Z
|
||||
|
||||
describe('liveMountXapiLabels', () => {
|
||||
it('keeps the name of the disk and names the SR after the restore point', () => {
|
||||
assert.deepEqual(liveMountXapiLabels({ timestamp: TIMESTAMP, vdiNameLabel: 'system', vmNameLabel: 'web01' }), {
|
||||
srNameLabel: '[XO live mount] web01 (20250801T080832Z)',
|
||||
vdiNameLabel: 'system',
|
||||
vdiNameDescription: 'Read-only live mount of web01 (20250801T080832Z), removed on unmount.',
|
||||
})
|
||||
})
|
||||
|
||||
it('says a live restore loses what is written to it', () => {
|
||||
const xapiLabels = liveMountXapiLabels({
|
||||
readWrite: true,
|
||||
timestamp: TIMESTAMP,
|
||||
vdiNameLabel: 'system',
|
||||
vmNameLabel: 'web01',
|
||||
})
|
||||
assert.equal(xapiLabels.srNameLabel, '[XO live restore] web01 (20250801T080832Z)')
|
||||
assert.match(xapiLabels.vdiNameDescription, /lost on unmount/)
|
||||
})
|
||||
})
|
||||
@@ -8,7 +8,7 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.78.0",
|
||||
"version": "0.76.0",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
},
|
||||
@@ -27,20 +27,19 @@
|
||||
"@vates/disposable": "^0.1.6",
|
||||
"@vates/fuse-vhd": "^2.1.3",
|
||||
"@vates/generator-toolbox": "^1.1.3",
|
||||
"@vates/nbd-client": "^3.7.0",
|
||||
"@vates/nbd-client": "^3.6.0",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@vates/task": "^0.7.0",
|
||||
"@vates/types": "^2.5.0",
|
||||
"@vates/types": "^2.3.0",
|
||||
"@xen-orchestra/async-map": "^0.1.5",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4",
|
||||
"@xen-orchestra/fs": "^4.10.0",
|
||||
"@xen-orchestra/disk-transform": "^1.3.3",
|
||||
"@xen-orchestra/fs": "^4.9.5",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"@xen-orchestra/qcow2": "^1.4.1",
|
||||
"@xen-orchestra/qcow2": "^1.4.0",
|
||||
"@xen-orchestra/template": "^0.1.1",
|
||||
"@xen-orchestra/backup-archive": "^1.2.0",
|
||||
"@xen-orchestra/backup-archive": "^1.1.3",
|
||||
"app-conf": "^3.0.0",
|
||||
"compare-versions": "^6.0.0",
|
||||
"complex-matcher": "^1.1.1",
|
||||
"d3-time-format": "^4.1.0",
|
||||
"decorator-synchronized": "^0.6.0",
|
||||
"golike-defer": "^0.5.1",
|
||||
@@ -56,10 +55,8 @@
|
||||
"tar": "^7.5.19",
|
||||
"uuid": "^9.0.0",
|
||||
"value-matcher": "^0.2.0",
|
||||
"vhd-lib": "^4.17.0",
|
||||
"xen-api": "^5.1.0",
|
||||
"xo-common": "^0.11.0",
|
||||
"xo-remote-parser": "^2.0.0",
|
||||
"vhd-lib": "^4.16.2",
|
||||
"xen-api": "^5.0.1",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -70,7 +67,7 @@
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^9.1.0"
|
||||
"@xen-orchestra/xapi": "^9.0.1"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"preferGlobal": true,
|
||||
"dependencies": {
|
||||
"golike-defer": "^0.5.1",
|
||||
"xen-api": "^5.1.0"
|
||||
"xen-api": "^5.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/disk-cli",
|
||||
"version": "2.2.5",
|
||||
"version": "2.2.3",
|
||||
"description": "CLI tool to inspect and manage disks in backup repositories",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
@@ -20,13 +20,13 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@xen-orchestra/backups": "^0.78.0",
|
||||
"@xen-orchestra/backup-archive": "^1.2.0",
|
||||
"@xen-orchestra/disk-transform": "^1.3.4",
|
||||
"@xen-orchestra/fs": "^4.10.0",
|
||||
"@xen-orchestra/qcow2": "^1.4.1",
|
||||
"@xen-orchestra/backups": "^0.76.0",
|
||||
"@xen-orchestra/backup-archive": "^1.1.3",
|
||||
"@xen-orchestra/disk-transform": "^1.3.3",
|
||||
"@xen-orchestra/fs": "^4.9.5",
|
||||
"@xen-orchestra/qcow2": "^1.4.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"vhd-lib": "^4.17.0"
|
||||
"vhd-lib": "^4.16.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.3.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@xen-orchestra/disk-transform",
|
||||
"version": "1.3.4",
|
||||
"version": "1.3.3",
|
||||
"main": "./dist/index.mjs",
|
||||
"license": "MIT",
|
||||
"private": false,
|
||||
|
||||
@@ -9,7 +9,7 @@ async function consumeGenerator(source: AsyncGenerator) {
|
||||
return 'finished'
|
||||
}
|
||||
test('Disk class', async t => {
|
||||
const testRes = await t.test('synchronized fork disk ', async () => {
|
||||
const testRes = await t.test('synchronised fork disk ', async () => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 20
|
||||
const source = new DebugDisk({ nbBlocks, blockSize, fillRate: 100 })
|
||||
|
||||
@@ -12,11 +12,11 @@ export class ThrottledDisk extends DiskPassthrough {
|
||||
const generator = await this.source.buildDiskBlockGenerator()
|
||||
//throttle want to be able to know the length of the data
|
||||
async function* generatorWithLength() {
|
||||
for await (const block of generator) {
|
||||
// keep the other properties of the block (like the compression of a raw VHD directory block)
|
||||
for await (const { index, data } of generator) {
|
||||
yield {
|
||||
...block,
|
||||
length: block.data.length,
|
||||
index,
|
||||
data,
|
||||
length: data.length,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
import { DiskBlock } from './Disk.mjs'
|
||||
import { DebugDisk } from './utils/DebugDisk.mjs'
|
||||
import { ThrottledDisk } from './Throttled.mjs'
|
||||
|
||||
type TaggedDiskBlock = DiskBlock & { compression: string }
|
||||
|
||||
// a disk yielding blocks with an additional property, like the raw blocks of a VHD directory
|
||||
class TaggedDebugDisk extends DebugDisk {
|
||||
async readBlock(index: number): Promise<TaggedDiskBlock> {
|
||||
return { ...(await super.readBlock(index)), compression: 'brotli' }
|
||||
}
|
||||
}
|
||||
|
||||
test('ThrottledDisk keeps the other properties of the blocks', async () => {
|
||||
const blockSize = 1024
|
||||
const nbBlocks = 5
|
||||
const source = new TaggedDebugDisk({ nbBlocks, blockSize, fillRate: 100 })
|
||||
await source.init()
|
||||
// speed 0 disables the throttling
|
||||
const disk = new ThrottledDisk(source, new Throttle(0))
|
||||
|
||||
let nbBlocksRead = 0
|
||||
for await (const block of disk.diskBlocks()) {
|
||||
assert.strictEqual((block as TaggedDiskBlock).compression, 'brotli')
|
||||
assert.strictEqual(block.data.length, blockSize)
|
||||
nbBlocksRead++
|
||||
}
|
||||
assert.strictEqual(nbBlocksRead, nbBlocks)
|
||||
})
|
||||
@@ -18,7 +18,7 @@ ee.on('start', async function () {
|
||||
// listeners have settled
|
||||
await ee.emitAsync('start')
|
||||
|
||||
// by default, it will rejects as soon as one listener reject, you can customize
|
||||
// by default, it will rejects as soon as one listener reject, you can customise
|
||||
// error handling though:
|
||||
await ee.emitAsync(
|
||||
{
|
||||
|
||||
@@ -36,7 +36,7 @@ ee.on('start', async function () {
|
||||
// listeners have settled
|
||||
await ee.emitAsync('start')
|
||||
|
||||
// by default, it will rejects as soon as one listener reject, you can customize
|
||||
// by default, it will rejects as soon as one listener reject, you can customise
|
||||
// error handling though:
|
||||
await ee.emitAsync(
|
||||
{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/fs",
|
||||
"version": "4.10.0",
|
||||
"version": "4.9.5",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "The File System for Xen Orchestra backups.",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
|
||||
|
||||
@@ -28,7 +28,6 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
|
||||
ivLength: 0,
|
||||
authTagLength: 0,
|
||||
encryptData: buffer => buffer,
|
||||
encryptDataParts: buffers => buffers,
|
||||
encryptStream: stream => stream,
|
||||
decryptData: buffer => buffer,
|
||||
decryptStream: stream => stream,
|
||||
@@ -110,17 +109,11 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
|
||||
)
|
||||
}
|
||||
|
||||
// data can be an array of buffers, encrypted as their concatenation
|
||||
// returns the encrypted file as an array of buffers, to be written one after the other
|
||||
function encryptDataParts(data) {
|
||||
function encryptData(buffer) {
|
||||
const iv = crypto.randomBytes(ivLength)
|
||||
const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
|
||||
const encrypted = Array.isArray(data) ? data.map(buffer => cipher.update(buffer)) : [cipher.update(data)]
|
||||
return [iv, ...encrypted, cipher.final(), authTagLength > 0 ? cipher.getAuthTag() : Buffer.alloc(0)]
|
||||
}
|
||||
|
||||
function encryptData(data) {
|
||||
return Buffer.concat(encryptDataParts(data))
|
||||
const encrypted = cipher.update(buffer)
|
||||
return Buffer.concat([iv, encrypted, cipher.final(), authTagLength > 0 ? cipher.getAuthTag() : Buffer.alloc(0)])
|
||||
}
|
||||
|
||||
function decryptData(buffer) {
|
||||
@@ -135,10 +128,7 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
|
||||
encrypted = buffer.slice(ivLength)
|
||||
}
|
||||
const decrypted = decipher.update(encrypted)
|
||||
// final() must always be called: it checks the auth tag
|
||||
const final = decipher.final()
|
||||
// nothing is left with stream modes (like GCM): no need to copy the data
|
||||
return final.length === 0 ? decrypted : Buffer.concat([decrypted, final])
|
||||
return Buffer.concat([decrypted, decipher.final()])
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -148,7 +138,6 @@ function getEncryptor(algorithm = DEFAULT_ENCRYPTION_ALGORITHM, key) {
|
||||
authTagLength,
|
||||
ivLength,
|
||||
encryptData,
|
||||
encryptDataParts,
|
||||
encryptStream,
|
||||
decryptData,
|
||||
decryptStream,
|
||||
|
||||
@@ -36,16 +36,6 @@ algorithms.forEach(algorithm => {
|
||||
assert.equal(decrypted.equals(buffer), true)
|
||||
})
|
||||
|
||||
it('handle an array of buffers', () => {
|
||||
// like a VHD block: a bitmap then the data, sizes not aligned on the cipher block size
|
||||
const parts = [crypto.randomBytes(512), crypto.randomBytes(1024 * 1024 + 1)]
|
||||
const encrypted = Buffer.concat(encryptor.encryptDataParts(parts))
|
||||
|
||||
// same layout as a single buffer: existing remotes must still be readable
|
||||
const decrypted = encryptor.decryptData(encrypted)
|
||||
assert.equal(decrypted.equals(Buffer.concat(parts)), true)
|
||||
})
|
||||
|
||||
it('handle stream', async () => {
|
||||
const stream = Readable.from(buffer)
|
||||
stream.length = buffer.length
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user