From c12bd807edbfe9ecf5bf0953b28e163e87a58de8 Mon Sep 17 00:00:00 2001 From: Florent BEAUCHAMP Date: Wed, 22 Jul 2026 14:36:50 +0200 Subject: [PATCH] feat(qa-test): add a script to do a load test (#10119) add new option and script to launch a load test : number of vm is configurable concurrency is configurable number of backup run is configurable will do multiple run, then a restore --- @xen-orchestra/qa-test/.USAGE.md | 63 +++- @xen-orchestra/qa-test/.env.example | 11 + @xen-orchestra/qa-test/.gitignore | 1 + @xen-orchestra/qa-test/README.md | 93 +++-- @xen-orchestra/qa-test/backup.config.js | 26 ++ .../qa-test/client/cleanupClient.js | 3 +- .../qa-test/client/requests/backup.js | 67 ++-- @xen-orchestra/qa-test/client/requests/vm.js | 20 + @xen-orchestra/qa-test/package.json | 5 +- .../qa-test/scripts/backup-load-delta.mjs | 357 ++++++++++++++++++ @xen-orchestra/qa-test/utils/backupUtils.js | 25 ++ @xen-orchestra/qa-test/utils/fleetUtils.js | 44 +++ @xen-orchestra/qa-test/utils/index.js | 66 ++++ @xen-orchestra/qa-test/utils/vmChurn.js | 55 +++ 14 files changed, 774 insertions(+), 62 deletions(-) create mode 100644 @xen-orchestra/qa-test/.gitignore create mode 100644 @xen-orchestra/qa-test/scripts/backup-load-delta.mjs create mode 100644 @xen-orchestra/qa-test/utils/fleetUtils.js create mode 100644 @xen-orchestra/qa-test/utils/vmChurn.js diff --git a/@xen-orchestra/qa-test/.USAGE.md b/@xen-orchestra/qa-test/.USAGE.md index 4818a4ca7c..c2b6a6575f 100644 --- a/@xen-orchestra/qa-test/.USAGE.md +++ b/@xen-orchestra/qa-test/.USAGE.md @@ -49,6 +49,14 @@ The following are required only for mirror backup tests (`qa:mirror`): | `MIRROR_DESTINATION_REPOSITORY_NAME` | Mirror destination repository name | `Test mirror QA` | | `MIRROR_DESTINATION_REPOSITORY_URL` | `@xen-orchestra/fs` URL for the mirror destination. Same backend flexibility as `BACKUP_REPOSITORY_URL`. | `file:///tmp/xo-test-mirror` | +The following are optional, for load tests (disk churn via SSH). Leave `TEST_VM_SSH_KEY` unset +to let `ssh` fall back to its own default identity files / `ssh-agent`: + +| Variable | Description | Example | +| ------------------ | ----------------------------------------------------------------------------------------------------------- | --------------- | +| `TEST_VM_SSH_USER` | SSH user on the load-test VMs (default `root`) | `root` | +| `TEST_VM_SSH_KEY` | Path to the private key matching a public key already authorized on `REFERENCE_VM_ID` (e.g. via cloud-init) | `~/.ssh/qa_key` | + > **Safety**: for `file://` remotes the path component of the URL must contain `test`, `qa`, or `tmp/xo` to prevent accidental deletion of production data. Non-local remotes (`s3://`, `nfs://`, …) skip this local-path check — cleanup is delegated to the XO API. ## Logging @@ -68,14 +76,15 @@ All debug-level logs are also written to a temp file at startup regardless of `D Log namespaces follow the pattern `qa:[:]`: -| Namespace | Test file | -| -------------------- | ------------------------------------------- | -| `qa:infrastructure` | `tests/infrastructure.test.js` | -| `qa:backup:base` | `tests/backup.test.js` | -| `qa:backup:nbd` | `tests/backup.nbd.test.js` | -| `qa:backup:combined` | `tests/backup-replication-combined.test.js` | -| `qa:mirror` | `tests/backup-mirror.test.js` | -| `qa:export` | `tests/export.vhd.test.js` | +| Namespace | Test file | +| ---------------------- | ------------------------------------------- | +| `qa:infrastructure` | `tests/infrastructure.test.js` | +| `qa:backup:base` | `tests/backup.test.js` | +| `qa:backup:nbd` | `tests/backup.nbd.test.js` | +| `qa:backup:combined` | `tests/backup-replication-combined.test.js` | +| `qa:mirror` | `tests/backup-mirror.test.js` | +| `qa:export` | `tests/export.vhd.test.js` | +| `qa:load:backup:delta` | `scripts/backup-load-delta.mjs` | ## Running tests @@ -105,6 +114,37 @@ yarn workspace @xen-orchestra/qa-test qa:export:xva # XVA export onl yarn workspace @xen-orchestra/qa-test qa:export:restore # Restoration only ``` +### Load tests + +Unlike the suites above, load tests are standalone scripts (not `node --test` files): they clone a +fleet of VMs from `REFERENCE_VM_ID`, churn their disks over SSH between runs, and repeatedly execute +one backup job spanning the whole fleet — measuring total job duration (export + merge/cleanup) per +run to help answer questions like "what's the optimal concurrency/retention for my setup?". + +```bash +yarn workspace @xen-orchestra/qa-test qa:load:backup:delta -- \ + --vms 16 --concurrency 4 --churn-percent 5 --runs 10 --retention 3 +``` + +| Flag | Default | Description | +| ----------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--vms` | `4` | Number of VMs cloned from `REFERENCE_VM_ID` into the fleet | +| `--concurrency` | `2` | Job's `concurrency` setting — how many VMs export in parallel; also bounds fleet clone/boot and per-run churn parallelism | +| `--churn-percent` | `5` | % of each VM's primary disk overwritten with fresh data before every run | +| `--runs` | `5` | Number of backup runs | +| `--retention` | `2` | Job's `exportRetention` — restore points kept before a merge is triggered. Set `< runs` or no merge ever fires | +| `--repositories` | `BACKUP_REPOSITORY_NAME` | Comma-separated backup repository name(s) to write to. The default name is auto-created from `BACKUP_REPOSITORY_URL` if missing; any other name must already exist in XO | +| `--output-dir` | `load-test-results//` | Where per-run JSON logs and `summary.json` are written | +| `--keep` | off | Skip all cleanup — fleet, job, schedule and backups are left in place | +| `--keep-backups` | off | Delete only the fleet VMs; leave the job, schedule and backup data in place (e.g. to inspect a dedup backend's on-disk state afterward) | + +Requires `SR_ID` (used to restore the latest backup of one fleet VM at the end of the run, validating +the chain is actually restorable) and optionally `TEST_VM_SSH_USER`/`TEST_VM_SSH_KEY` (see above). + +Each run's full backup log is written to `/run-.json`; `summary.json` has per-run +timing/throughput plus fleet-wide totals — feed either back for analysis the same way as any other +backup log dump. + ### Demo (quick connectivity check) ```bash @@ -143,10 +183,13 @@ yarn workspace @xen-orchestra/qa-test demo │ ├── index.js # Assertions and utilities (waitUntil, scheduling) │ ├── backupUtils.js # Backup validation utilities │ ├── exportUtils.js # VHD/XVA integrity validation -│ └── resourceTracker.js # Resource tracking for automatic cleanup +│ ├── resourceTracker.js # Resource tracking for automatic cleanup +│ ├── vmChurn.js # SSH-based disk churn for load tests +│ └── fleetUtils.js # Clone/boot a fleet of VMs for load tests └── scripts/ ├── test-backup-and-purge.js # Standalone script: backup + purge E2E - └── test-purge-backups.js # Standalone script: purge diagnostics + ├── test-purge-backups.js # Standalone script: purge diagnostics + └── backup-load-delta.mjs # Load test: delta backup on a cloned VM fleet ``` ### Dispatch pattern diff --git a/@xen-orchestra/qa-test/.env.example b/@xen-orchestra/qa-test/.env.example index 15d60970e7..4395deec09 100644 --- a/@xen-orchestra/qa-test/.env.example +++ b/@xen-orchestra/qa-test/.env.example @@ -39,6 +39,17 @@ MIRROR_DESTINATION_REPOSITORY_URL="file:///tmp/xo-test-mirror" # Must be a different SR from where the source VMs live REPLICATION_DESTINATION_SR_ID= # Required: UUID of the destination Storage Repository +# =========================================== +# LOAD TEST CONFIGURATION +# =========================================== + +# SSH access to cloned load-test VMs, used to churn disk content between backup runs. +# A public key must already be authorized on REFERENCE_VM_ID (e.g. baked in via cloud-init) +# so every clone inherits it. Both vars are optional: leave TEST_VM_SSH_KEY unset to let ssh +# use its own default identity files / ssh-agent instead of an explicit -i. +TEST_VM_SSH_USER=root +TEST_VM_SSH_KEY= # Optional: path to the private key authorized on the reference VM + # =========================================== # SDN CONTROLLER TEST CONFIGURATION # =========================================== diff --git a/@xen-orchestra/qa-test/.gitignore b/@xen-orchestra/qa-test/.gitignore new file mode 100644 index 0000000000..11bb9fcc73 --- /dev/null +++ b/@xen-orchestra/qa-test/.gitignore @@ -0,0 +1 @@ +/load-test-results/ diff --git a/@xen-orchestra/qa-test/README.md b/@xen-orchestra/qa-test/README.md index 1069dd3e05..dcb2ab63dd 100644 --- a/@xen-orchestra/qa-test/README.md +++ b/@xen-orchestra/qa-test/README.md @@ -38,24 +38,32 @@ cp .env.example .env All variables are required. The test suite fails immediately if any are missing. -| Variable | Description | Example | -| ------------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------- | -| `HOSTNAME` | XO instance URL | `http://10.1.4.216:9000` | -| `USERNAME` | XO user | `admin@admin.net` | -| `PASSWORD` | XO password | `admin` | -| `REFERENCE_VM_ID` | UUID of the VM to clone for tests | `61c24db9-262d-...` | -| `SR_ID` | UUID of the Storage Repository | `8aa2fb4a-143e-...` | -| `VM_PREFIX` | Test VM name prefix | `TST` | -| `BACKUP_REPOSITORY_NAME` | Backup repository name in XO | `Test backup QA` | -| `BACKUP_REPOSITORY_URL` | `@xen-orchestra/fs` URL for the test backup remote. For `file://` remotes the path must contain `test`, `qa`, or `tmp/xo`. Any supported backend works (`s3://`, `nfs://`, `azure://`, …). | `file:///tmp/xo-test-backups` | -| `VHD_EXPORT_PATH` | VHD/XVA export directory (must contain `test`, `qa`, or `tmp/`) | `/tmp/xo-test-exports` | +| Variable | Description | Example | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------- | +| `HOSTNAME` | XO instance URL | `http://10.1.4.216:9000` | +| `USERNAME` | XO user | `admin@admin.net` | +| `PASSWORD` | XO password | `admin` | +| `REFERENCE_VM_ID` | UUID of the VM to clone for tests | `61c24db9-262d-...` | +| `SR_ID` | UUID of the Storage Repository | `8aa2fb4a-143e-...` | +| `VM_PREFIX` | Test VM name prefix | `TST` | +| `BACKUP_REPOSITORY_NAME` | Backup repository name in XO | `Test backup QA` | +| `BACKUP_REPOSITORY_URL` | `@xen-orchestra/fs` URL for the test backup remote. For `file://` remotes the path must contain `test`, `qa`, or `tmp/xo`. Any supported backend works (`s3://`, `nfs://`, `azure://`, …). | `file:///tmp/xo-test-backups` | +| `VHD_EXPORT_PATH` | VHD/XVA export directory (must contain `test`, `qa`, or `tmp/`) | `/tmp/xo-test-exports` | The following are required only for mirror backup tests (`qa:mirror`): -| Variable | Description | Example | -| ------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ----------------------------------- | -| `MIRROR_DESTINATION_REPOSITORY_NAME` | Mirror destination repository name | `Test mirror QA` | -| `MIRROR_DESTINATION_REPOSITORY_URL` | `@xen-orchestra/fs` URL for the mirror destination. Same backend flexibility as `BACKUP_REPOSITORY_URL`. | `file:///tmp/xo-test-mirror` | +| Variable | Description | Example | +| ------------------------------------ | -------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `MIRROR_DESTINATION_REPOSITORY_NAME` | Mirror destination repository name | `Test mirror QA` | +| `MIRROR_DESTINATION_REPOSITORY_URL` | `@xen-orchestra/fs` URL for the mirror destination. Same backend flexibility as `BACKUP_REPOSITORY_URL`. | `file:///tmp/xo-test-mirror` | + +The following are optional, for load tests (disk churn via SSH). Leave `TEST_VM_SSH_KEY` unset +to let `ssh` fall back to its own default identity files / `ssh-agent`: + +| Variable | Description | Example | +| ------------------ | ----------------------------------------------------------------------------------------------------------- | --------------- | +| `TEST_VM_SSH_USER` | SSH user on the load-test VMs (default `root`) | `root` | +| `TEST_VM_SSH_KEY` | Path to the private key matching a public key already authorized on `REFERENCE_VM_ID` (e.g. via cloud-init) | `~/.ssh/qa_key` | > **Safety**: for `file://` remotes the path component of the URL must contain `test`, `qa`, or `tmp/xo` to prevent accidental deletion of production data. Non-local remotes (`s3://`, `nfs://`, …) skip this local-path check — cleanup is delegated to the XO API. @@ -76,14 +84,15 @@ All debug-level logs are also written to a temp file at startup regardless of `D Log namespaces follow the pattern `qa:[:]`: -| Namespace | Test file | -| -------------------- | ------------------------------------------- | -| `qa:infrastructure` | `tests/infrastructure.test.js` | -| `qa:backup:base` | `tests/backup.test.js` | -| `qa:backup:nbd` | `tests/backup.nbd.test.js` | -| `qa:backup:combined` | `tests/backup-replication-combined.test.js` | -| `qa:mirror` | `tests/backup-mirror.test.js` | -| `qa:export` | `tests/export.vhd.test.js` | +| Namespace | Test file | +| ---------------------- | ------------------------------------------- | +| `qa:infrastructure` | `tests/infrastructure.test.js` | +| `qa:backup:base` | `tests/backup.test.js` | +| `qa:backup:nbd` | `tests/backup.nbd.test.js` | +| `qa:backup:combined` | `tests/backup-replication-combined.test.js` | +| `qa:mirror` | `tests/backup-mirror.test.js` | +| `qa:export` | `tests/export.vhd.test.js` | +| `qa:load:backup:delta` | `scripts/backup-load-delta.mjs` | ## Running tests @@ -113,6 +122,37 @@ yarn workspace @xen-orchestra/qa-test qa:export:xva # XVA export onl yarn workspace @xen-orchestra/qa-test qa:export:restore # Restoration only ``` +### Load tests + +Unlike the suites above, load tests are standalone scripts (not `node --test` files): they clone a +fleet of VMs from `REFERENCE_VM_ID`, churn their disks over SSH between runs, and repeatedly execute +one backup job spanning the whole fleet — measuring total job duration (export + merge/cleanup) per +run to help answer questions like "what's the optimal concurrency/retention for my setup?". + +```bash +yarn workspace @xen-orchestra/qa-test qa:load:backup:delta -- \ + --vms 16 --concurrency 4 --churn-percent 5 --runs 10 --retention 3 +``` + +| Flag | Default | Description | +| ----------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--vms` | `4` | Number of VMs cloned from `REFERENCE_VM_ID` into the fleet | +| `--concurrency` | `2` | Job's `concurrency` setting — how many VMs export in parallel; also bounds fleet clone/boot and per-run churn parallelism | +| `--churn-percent` | `5` | % of each VM's primary disk overwritten with fresh data before every run | +| `--runs` | `5` | Number of backup runs | +| `--retention` | `2` | Job's `exportRetention` — restore points kept before a merge is triggered. Set `< runs` or no merge ever fires | +| `--repositories` | `BACKUP_REPOSITORY_NAME` | Comma-separated backup repository name(s) to write to. The default name is auto-created from `BACKUP_REPOSITORY_URL` if missing; any other name must already exist in XO | +| `--output-dir` | `load-test-results//` | Where per-run JSON logs and `summary.json` are written | +| `--keep` | off | Skip all cleanup — fleet, job, schedule and backups are left in place | +| `--keep-backups` | off | Delete only the fleet VMs; leave the job, schedule and backup data in place (e.g. to inspect a dedup backend's on-disk state afterward) | + +Requires `SR_ID` (used to restore the latest backup of one fleet VM at the end of the run, validating +the chain is actually restorable) and optionally `TEST_VM_SSH_USER`/`TEST_VM_SSH_KEY` (see above). + +Each run's full backup log is written to `/run-.json`; `summary.json` has per-run +timing/throughput plus fleet-wide totals — feed either back for analysis the same way as any other +backup log dump. + ### Demo (quick connectivity check) ```bash @@ -151,10 +191,13 @@ yarn workspace @xen-orchestra/qa-test demo │ ├── index.js # Assertions and utilities (waitUntil, scheduling) │ ├── backupUtils.js # Backup validation utilities │ ├── exportUtils.js # VHD/XVA integrity validation -│ └── resourceTracker.js # Resource tracking for automatic cleanup +│ ├── resourceTracker.js # Resource tracking for automatic cleanup +│ ├── vmChurn.js # SSH-based disk churn for load tests +│ └── fleetUtils.js # Clone/boot a fleet of VMs for load tests └── scripts/ ├── test-backup-and-purge.js # Standalone script: backup + purge E2E - └── test-purge-backups.js # Standalone script: purge diagnostics + ├── test-purge-backups.js # Standalone script: purge diagnostics + └── backup-load-delta.mjs # Load test: delta backup on a cloned VM fleet ``` ### Dispatch pattern diff --git a/@xen-orchestra/qa-test/backup.config.js b/@xen-orchestra/qa-test/backup.config.js index d2b163bfde..6ce63d43b9 100644 --- a/@xen-orchestra/qa-test/backup.config.js +++ b/@xen-orchestra/qa-test/backup.config.js @@ -50,3 +50,29 @@ export function backupConfig(name, schedule, vm, backupRepository, options = {}) return config } + +// One delta job over a whole fleet (a future fleetFullBackupConfig covers mode: 'full'). +// mergeBackupsSynchronously folds merge/cleanup into the job's own duration, so start/end +// alone is the total cost — nothing extra to measure. +export function fleetDeltaBackupConfig(name, schedule, vmIds, backupRepositoryIds, options = {}) { + const { concurrency = 2, exportRetention = 2 } = options + + return { + name, + mode: 'delta', + schedules: { + '': schedule, + }, + settings: { + '': { + timezone: 'Europe/Paris', + concurrency, + exportRetention, + mergeBackupsSynchronously: true, + bypassVdiChainsCheck: true, + }, + }, + vms: Object.fromEntries(vmIds.map(id => [id, true])), + remotes: Object.fromEntries(backupRepositoryIds.map(id => [id, true])), + } +} diff --git a/@xen-orchestra/qa-test/client/cleanupClient.js b/@xen-orchestra/qa-test/client/cleanupClient.js index 6878bde1d5..c1d37c0016 100644 --- a/@xen-orchestra/qa-test/client/cleanupClient.js +++ b/@xen-orchestra/qa-test/client/cleanupClient.js @@ -2,6 +2,7 @@ import { createLogger } from '@xen-orchestra/log' import path from 'node:path' import fs from 'node:fs/promises' import { FilterBuilder } from './FilterBuilder.js' +import { extractIdsFromSimplePattern } from '@xen-orchestra/backups/extractIdsFromSimplePattern.mjs' import { BACKUP_JOB_NAME_PREFIX, getRequiredEnv } from '../utils/index.js' const log = createLogger('cleanup') @@ -279,7 +280,7 @@ export class CleanupClient { if (config.deleteBackupFiles) { try { const jobDetails = await this.dispatchClient.backup.details(job.id) - const jobRemotes = jobDetails.remotes?.id ? [jobDetails.remotes.id] : Object.keys(jobDetails.remotes || {}) + const jobRemotes = extractIdsFromSimplePattern(jobDetails.remotes) if (jobRemotes.length > 0) { const backupsByRemote = await this.dispatchClient.backup.listVmBackups(jobRemotes) diff --git a/@xen-orchestra/qa-test/client/requests/backup.js b/@xen-orchestra/qa-test/client/requests/backup.js index 8e6329048a..7daef5e691 100644 --- a/@xen-orchestra/qa-test/client/requests/backup.js +++ b/@xen-orchestra/qa-test/client/requests/backup.js @@ -1,5 +1,5 @@ import { createLogger } from '@xen-orchestra/log' -import { waitUntil } from '../../utils/index.js' +import { toSimplePattern, waitUntil } from '../../utils/index.js' import { AbstractRequest } from './abstract.js' const log = createLogger('xo:qa-test:backup') @@ -39,30 +39,24 @@ export class BackupRequest extends AbstractRequest { throw new Error('Valid backup configuration object is required') } - // Convert dynamic key format to XO expected format const convertConfig = cfg => { const { vms, remotes, srs, ...rest } = cfg - // Extract first VM UUID - const vmUuid = vms && typeof vms === 'object' ? Object.keys(vms)[0] : undefined + const result = { ...rest } - const result = { - ...rest, - vms: { - id: vmUuid, - }, + const vmIds = vms && typeof vms === 'object' ? Object.keys(vms) : [] + if (vmIds.length > 0) { + result.vms = toSimplePattern(vmIds) } - // Extract first Backup Repository ID (for backup to remote) - const backupRepositoryId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined - if (backupRepositoryId !== undefined) { - result.remotes = { id: backupRepositoryId } + const backupRepositoryIds = remotes && typeof remotes === 'object' ? Object.keys(remotes) : [] + if (backupRepositoryIds.length > 0) { + result.remotes = toSimplePattern(backupRepositoryIds) } - // Extract first SR ID (for CR/DR mode — replication to SR) - const srId = srs && typeof srs === 'object' ? Object.keys(srs)[0] : undefined - if (srId !== undefined) { - result.srs = { id: srId } + const srIds = srs && typeof srs === 'object' ? Object.keys(srs) : [] + if (srIds.length > 0) { + result.srs = toSimplePattern(srIds) } return result @@ -165,6 +159,29 @@ export class BackupRequest extends AbstractRequest { } } + /** + * Restores a VM backup to a new VM on the given SR (backupNg.importVmBackup). + * + * @param {string} backupId - Backup ID as returned by `listVmBackups` + * @param {string} srId - Target Storage Repository UUID + * @param {Object} [settings={}] + * @returns {Promise} UUID of the restored VM + */ + async importVmBackup(backupId, srId, settings = {}) { + this._ensureConnected() + + try { + return await this.dispatchClient.xoClient.call('backupNg.importVmBackup', { + id: backupId, + sr: srId, + settings, + }) + } catch (error) { + log.warn('Import VM backup failed', { error }) + throw error + } + } + // =========================================================================== // Mirror Backup operations (mirrorBackup.* API) // =========================================================================== @@ -194,16 +211,16 @@ export class BackupRequest extends AbstractRequest { const { remotes, ...rest } = config - // Convert remotes to { id: remoteId } format - const remoteId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined + // Convert destination remotes to a simple pattern (supports one or several) + const remoteIds = remotes && typeof remotes === 'object' ? Object.keys(remotes) : [] const xoConfig = { ...rest, mode: config.mode || 'full', } - if (remoteId !== undefined) { - xoConfig.remotes = { id: remoteId } + if (remoteIds.length > 0) { + xoConfig.remotes = toSimplePattern(remoteIds) } try { @@ -316,12 +333,12 @@ export class BackupRequest extends AbstractRequest { const { pools, remotes, ...rest } = config - const poolId = pools && typeof pools === 'object' ? Object.keys(pools)[0] : undefined - const remoteId = remotes && typeof remotes === 'object' ? Object.keys(remotes)[0] : undefined + const poolIds = pools && typeof pools === 'object' ? Object.keys(pools) : [] + const remoteIds = remotes && typeof remotes === 'object' ? Object.keys(remotes) : [] const xoConfig = { ...rest } - if (poolId !== undefined) xoConfig.pools = { id: poolId } - if (remoteId !== undefined) xoConfig.remotes = { id: remoteId } + if (poolIds.length > 0) xoConfig.pools = toSimplePattern(poolIds) + if (remoteIds.length > 0) xoConfig.remotes = toSimplePattern(remoteIds) try { const result = await this.dispatchClient.xoClient.call('metadataBackup.createJob', xoConfig) diff --git a/@xen-orchestra/qa-test/client/requests/vm.js b/@xen-orchestra/qa-test/client/requests/vm.js index 05865c7bae..95fc4d08e6 100644 --- a/@xen-orchestra/qa-test/client/requests/vm.js +++ b/@xen-orchestra/qa-test/client/requests/vm.js @@ -268,6 +268,26 @@ export class VMRequest extends AbstractRequest { ) } + // Guest metrics lag behind power_state === 'Running', so SSH-based callers must wait for this too. + async waitForGuestIp(vmUuid, timeout) { + assertNonEmptyString(vmUuid, 'Valid VM UUID is required', 'INVALID_VM_UUID') + this._ensureConnected() + + return waitUntil( + async () => { + const vm = await this.details(vmUuid) + // XO normalizes guest-metrics keys to `${device}/${protocol}/${index}` (e.g. '0/ipv4/0'), + // not the raw XAPI 'x/ip' field — match any device/index. waitUntil only keeps polling + // while the condition returns exactly `false`. + const addresses = vm?.addresses ?? {} + const ipv4Key = Object.keys(addresses).find(key => key.includes('/ipv4/')) + return (ipv4Key && addresses[ipv4Key]) || false + }, + 3_000, + timeout + ) + } + /** * Imports an XVA file to create a new VM. * diff --git a/@xen-orchestra/qa-test/package.json b/@xen-orchestra/qa-test/package.json index f5d58552fc..8b9c4696e7 100644 --- a/@xen-orchestra/qa-test/package.json +++ b/@xen-orchestra/qa-test/package.json @@ -20,6 +20,8 @@ "node": ">=18" }, "dependencies": { + "@vates/async-each": "^1.0.3", + "@xen-orchestra/backups": "^0.73.7", "@xen-orchestra/fs": "^4.9.2", "@xen-orchestra/log": "^0.7.2", "nodemailer": "^7.0.13", @@ -45,6 +47,7 @@ "demo": "node --env-file-if-exists=.env index.js", "report": "node --env-file-if-exists=.env scripts/run-and-report.mjs", "report:smtp-test": "node --env-file-if-exists=.env scripts/run-and-report.mjs --smtp-test", - "qa:backup:file-restore": "node --env-file-if-exists=.env --test tests/backup.file-restore.test.js" + "qa:backup:file-restore": "node --env-file-if-exists=.env --test tests/backup.file-restore.test.js", + "qa:load:backup:delta": "node --env-file-if-exists=.env scripts/backup-load-delta.mjs" } } diff --git a/@xen-orchestra/qa-test/scripts/backup-load-delta.mjs b/@xen-orchestra/qa-test/scripts/backup-load-delta.mjs new file mode 100644 index 0000000000..c50c4bb0e2 --- /dev/null +++ b/@xen-orchestra/qa-test/scripts/backup-load-delta.mjs @@ -0,0 +1,357 @@ +#!/usr/bin/env node +/** + * Delta (incremental) backup load test. + * + * Clones N VMs from REFERENCE_VM_ID into a single fleet, then repeatedly: + * 1. churns every VM's disk over SSH (a fixed % of its virtual size) + * 2. runs ONE delta backup job spanning the whole fleet + * 3. records the job's total duration and transferred bytes + * Once all runs are done, restores the latest backup of one fleet VM onto SR_ID to + * validate the chain is actually restorable, before the fleet/job get cleaned up. + * + * A sibling `backup-load-full.mjs` will follow the same shape for full backup jobs. + * + * The job uses `mergeBackupsSynchronously: true`, so once `--retention` is exceeded + * the VHD merge/cleanup for a run happens inside that run's own job duration instead + * of a background worker — the recorded duration is already export + merge, nothing + * extra to add up. + * + * Usage: + * node --env-file-if-exists=.env scripts/backup-load-delta.mjs \ + * --vms 16 --concurrency 4 --churn-percent 5 --runs 10 --retention 3 \ + * [--repositories [,...]] + * + * --repositories defaults to [BACKUP_REPOSITORY_NAME], created from BACKUP_REPOSITORY_URL + * if it doesn't exist yet. Extra names must already exist in XO (no URL to create them with). + * + * --keep leaves the whole fleet, job, schedule and backups in place (nothing is cleaned up). + * --keep-backups deletes only the fleet VMs, leaving the job/schedule/backup data untouched — + * for inspecting the repository's on-disk state afterward (e.g. dedup ratio on a dedup backend). + * + * Optional env: TEST_VM_SSH_USER (default 'root'), TEST_VM_SSH_KEY (private key path). + * Leave TEST_VM_SSH_KEY unset if the operator's default identity/ssh-agent is already + * authorized on REFERENCE_VM_ID — ssh will use it without a -i flag. + * + * Results are written to --output-dir (default: load-test-results//) as one + * JSON file per run plus a summary.json — feed them back for analysis the same way as + * any other backup log dump. + */ +import '../logSetup.js' +import { parseArgs } from 'node:util' +import { mkdir, writeFile } from 'node:fs/promises' +import { createLogger } from '@xen-orchestra/log' +import { asyncEach } from '@vates/async-each' +import { DispatchClient } from '../client/dispatchClient.js' +import { createResourceTracker } from '../utils/resourceTracker.js' +import { cloneFleet } from '../utils/fleetUtils.js' +import { churnVm, churnSizeMb } from '../utils/vmChurn.js' +import { fleetDeltaBackupConfig } from '../backup.config.js' +import { + getRequiredEnv, + getDefaultSchedule, + getScheduleKey, + generateBackupJobName, + formatDuration, + sumBackupTransferredBytes, + sumBackupPhaseDurations, +} from '../utils/index.js' +import { assertBackupSuccess, resolveOrCreateBackupRepository } from '../utils/backupUtils.js' + +const log = createLogger('qa:load:backup:delta') + +function parseCliArgs() { + const { values } = parseArgs({ + options: { + vms: { type: 'string', default: '4' }, + concurrency: { type: 'string', default: '2' }, + 'churn-percent': { type: 'string', default: '5' }, + runs: { type: 'string', default: '5' }, + retention: { type: 'string', default: '2' }, + repositories: { type: 'string' }, + 'output-dir': { type: 'string' }, + keep: { type: 'boolean', default: false }, + 'keep-backups': { type: 'boolean', default: false }, + }, + }) + + return { + vmCount: Number(values.vms), + concurrency: Number(values.concurrency), + churnPercent: Number(values['churn-percent']), + runs: Number(values.runs), + retention: Number(values.retention), + // Undefined means "use BACKUP_REPOSITORY_NAME", resolved (and auto-created if needed) in main(). + repositoryNames: values.repositories + ?.split(',') + .map(name => name.trim()) + .filter(Boolean), + outputDir: values['output-dir'], + keep: values.keep, + keepBackups: values['keep-backups'], + } +} + +// Reuses/creates the default (env-configured) repository; any other requested repository +// must already exist in XO, since only BACKUP_REPOSITORY_URL gives us a URL to create with. +async function resolveBackupRepositories({ dispatchClient, tracker, repositoryNames }) { + const defaultName = getRequiredEnv('BACKUP_REPOSITORY_NAME') + const names = repositoryNames ?? [defaultName] + + const repositories = [] + for (const name of names) { + if (name === defaultName) { + repositories.push( + await resolveOrCreateBackupRepository(dispatchClient, tracker, { + name, + url: getRequiredEnv('BACKUP_REPOSITORY_URL'), + }) + ) + continue + } + + const repository = await dispatchClient.backupRepository.get({ name }) + if (!repository) { + throw new Error(`Backup repository "${name}" not found — only BACKUP_REPOSITORY_NAME can be auto-created`) + } + repositories.push(repository) + } + + return repositories +} + +// Assumes a single meaningfully-sized data disk per VM (churns the largest user VDI). +async function resolveChurnTargets({ dispatchClient, fleet, churnPercent, concurrency }) { + const targets = [] + + await asyncEach( + fleet, + async ({ id, ip }) => { + const vdis = await dispatchClient.vdi.getVdisForVm(id) + if (vdis.length === 0) { + throw new Error(`VM ${id} has no user VDI to churn`, { cause: { vmId: id } }) + } + + const primaryVdi = vdis.reduce((largest, vdi) => (vdi.virtual_size > largest.virtual_size ? vdi : largest)) + targets.push({ id, ip, sizeMb: churnSizeMb(primaryVdi.virtual_size, churnPercent) }) + }, + { concurrency } + ) + + return targets +} + +// Validates the backup chain is actually restorable after all the churn/merge activity, +// not just that the job reported success. Picks one VM (fleet[0]) and its most recent backup. +async function restoreLatestBackup({ dispatchClient, tracker, fleet, backupRepositories, srId }) { + const vmId = fleet[0].id + const backupsByRemote = await dispatchClient.backup.listVmBackups(backupRepositories.map(r => r.id)) + + const repository = backupRepositories.find(r => backupsByRemote[r.id]?.[vmId]?.length > 0) + if (!repository) { + throw new Error(`No backups found for VM ${vmId} in any configured repository`) + } + + const backups = backupsByRemote[repository.id][vmId] + const latestBackup = [...backups].sort((a, b) => a.timestamp - b.timestamp).at(-1) + + const restoredVmId = await dispatchClient.backup.importVmBackup(latestBackup.id, srId) + tracker?.trackResource('restoredVm', restoredVmId, { sourceVmId: vmId, backupId: latestBackup.id }) + return restoredVmId +} + +async function main() { + const { vmCount, concurrency, churnPercent, runs, retention, repositoryNames, outputDir, keep, keepBackups } = + parseCliArgs() + + if (retention >= runs) { + log.warn( + 'exportRetention >= runs: no restore point will ever be pruned during this test, so no merge ' + + 'will be triggered and the total-duration measurement will not reflect merge/cleanup cost', + { retention, runs } + ) + } + + const dispatchClient = new DispatchClient() + await dispatchClient.initialize() + const tracker = createResourceTracker() + + try { + const setupStart = Date.now() + + const referenceVmId = getRequiredEnv('REFERENCE_VM_ID') + const referenceVm = await dispatchClient.vm.details(referenceVmId) + if (!referenceVm) { + throw new Error(`Reference VM ${referenceVmId} not found`) + } + + const backupRepositories = await resolveBackupRepositories({ dispatchClient, tracker, repositoryNames }) + + // Both optional: unset falls back to ssh's own default identity files / ssh-agent. + const sshUser = process.env.TEST_VM_SSH_USER || 'root' + const sshKey = process.env.TEST_VM_SSH_KEY + + const name = generateBackupJobName() + const outputPath = outputDir ?? `load-test-results/${name}` + await mkdir(outputPath, { recursive: true }) + + log.info('Cloning fleet', { vmCount, concurrency }) + const fleet = await cloneFleet({ + dispatchClient, + referenceVm, + count: vmCount, + namePrefix: `${getRequiredEnv('VM_PREFIX')}-QA-Load-${tracker.getSessionId()}`, + tracker, + concurrency, + }) + + const churnTargets = await resolveChurnTargets({ dispatchClient, fleet, churnPercent, concurrency }) + + const schedule = getDefaultSchedule() + const jobConfig = fleetDeltaBackupConfig( + name, + schedule, + fleet.map(vm => vm.id), + backupRepositories.map(repository => repository.id), + { concurrency, exportRetention: retention } + ) + + const jobId = await dispatchClient.backup.createBackupJob(jobConfig) + tracker.trackResource('backupJob', jobId, { name }) + const job = await dispatchClient.backup.details(jobId) + const scheduleId = getScheduleKey(job) + tracker.trackResource('schedule', scheduleId, { name, jobId }) + + // Fleet clone/boot/IP-wait dominates this and is excluded from per-run job timings below. + const setupDurationMs = Date.now() - setupStart + log.info('Fleet + job setup complete', { duration: formatDuration(setupDurationMs) }) + + const results = [] + for (let run = 1; run <= runs; run++) { + log.info(`Run ${run}/${runs}: churning fleet`, { vmCount, churnPercent }) + const churnStart = Date.now() + await asyncEach( + churnTargets, + target => churnVm({ ip: target.ip, identityFile: sshKey, user: sshUser, sizeMb: target.sizeMb }), + { concurrency } + ) + const churnDurationMs = Date.now() - churnStart + + log.info(`Run ${run}/${runs}: running backup job`, { jobId, scheduleId }) + const result = await dispatchClient.backup.runJobAndGetLog(jobId, scheduleId) + assertBackupSuccess(result, `Load test run ${run}/${runs}`) + + const jobDurationMs = result.end - result.start + const transferredBytes = sumBackupTransferredBytes(result) + const { snapshotDurationMs, cleanVmBeforeDurationMs, cleanVmAfterDurationMs } = sumBackupPhaseDurations(result) + const summary = { + run, + churnDurationMs, + jobDurationMs, + snapshotDurationMs, + cleanVmBeforeDurationMs, + cleanVmAfterDurationMs, + transferredBytes, + throughputMBps: transferredBytes / 1024 / 1024 / (jobDurationMs / 1000), + } + results.push(summary) + + await writeFile(`${outputPath}/run-${run}.json`, JSON.stringify(result, null, 2)) + log.info(`Run ${run}/${runs} done`, { + churnDuration: formatDuration(churnDurationMs), + duration: formatDuration(jobDurationMs), + snapshotDuration: formatDuration(snapshotDurationMs), + cleanVmAfterDuration: formatDuration(cleanVmAfterDurationMs), + transferredGiB: (transferredBytes / 1024 ** 3).toFixed(2), + throughputMBps: summary.throughputMBps.toFixed(1), + }) + } + + const sumField = field => results.reduce((total, result) => total + result[field], 0) + const runDurationMs = sumField('jobDurationMs') + const totalChurnDurationMs = sumField('churnDurationMs') + const totalSnapshotDurationMs = sumField('snapshotDurationMs') + const totalCleanVmBeforeDurationMs = sumField('cleanVmBeforeDurationMs') + const totalCleanVmAfterDurationMs = sumField('cleanVmAfterDurationMs') + const totalTransferredBytes = sumField('transferredBytes') + + const srId = getRequiredEnv('SR_ID') + log.info('Restoring latest backup to validate the chain', { srId }) + const restoreStart = Date.now() + const restoredVmId = await restoreLatestBackup({ dispatchClient, tracker, fleet, backupRepositories, srId }) + const restoreDurationMs = Date.now() - restoreStart + log.info('Restore validation complete', { restoredVmId, duration: formatDuration(restoreDurationMs) }) + + await writeFile( + `${outputPath}/summary.json`, + JSON.stringify( + { + vmCount, + concurrency, + churnPercent, + retention, + runs, + setupDurationMs, + runDurationMs, + restoreDurationMs, + totalChurnDurationMs, + totalSnapshotDurationMs, + totalCleanVmBeforeDurationMs, + totalCleanVmAfterDurationMs, + totalTransferredBytes, + results, + }, + null, + 2 + ) + ) + log.info('Load test complete', { outputPath, runDuration: formatDuration(runDurationMs) }) + } finally { + if (!keep) { + log.info( + keepBackups + ? 'Cleaning up load-test VMs (--keep-backups: leaving job, schedule and backup data in place)' + : 'Cleaning up load-test resources' + ) + const cleanupStart = Date.now() + const trackedResources = tracker.getTrackedResources() + try { + await dispatchClient.cleanup.fullCleanup({ + cleanupVMs: true, + cleanupBackupJobs: !keepBackups, + cleanupSchedules: !keepBackups, + // Only deletes the repository if resolveOrCreateBackupRepository tracked one it created — + // a reused pre-existing repository (the common case) is left untouched. Never deleted with + // --keep-backups, since that removes the physical backup files along with the config. + backupRepositoryId: keepBackups ? null : trackedResources.backupRepository?.id || null, + additionalVmIds: trackedResources.vms.map(vm => vm.id), + additionalJobIds: trackedResources.backupJobs.map(job => job.id), + additionalScheduleIds: trackedResources.schedules.map(schedule => schedule.id), + }) + } catch (error) { + log.warn('Fleet/job cleanup failed', { error }) + } + + // Not part of fullCleanup: the VM restored for chain validation. + try { + await dispatchClient.cleanup.deleteRestoredVMs({ vmIds: trackedResources.restoredVms.map(vm => vm.id) }) + } catch (error) { + log.warn('Restored-VM cleanup failed', { error }) + } + + log.info('Cleanup complete', { duration: formatDuration(Date.now() - cleanupStart) }) + } else { + log.info('Skipping cleanup (--keep)') + } + + try { + await dispatchClient.close() + } catch (error) { + log.warn('Failed to close connections', { error }) + } + } +} + +main().catch(error => { + log.warn('Load test failed', { error }) + process.exitCode = 1 +}) diff --git a/@xen-orchestra/qa-test/utils/backupUtils.js b/@xen-orchestra/qa-test/utils/backupUtils.js index 3a31d149c3..38464261b0 100644 --- a/@xen-orchestra/qa-test/utils/backupUtils.js +++ b/@xen-orchestra/qa-test/utils/backupUtils.js @@ -1,5 +1,6 @@ import { createLogger } from '@xen-orchestra/log' import assert from 'node:assert' +import { getSyncedHandler } from '@xen-orchestra/fs' const log = createLogger('xo:qa-test:backup-utils') @@ -60,6 +61,30 @@ export const assertRepositoryMatchesConfig = (repository, expectedUrl) => { } } +// Same find-or-create logic as tests/setup.js, factored out for reuse by load tests. +// Only tracks (for teardown) the repository if this call is the one that created it. +export async function resolveOrCreateBackupRepository(dispatchClient, tracker, { name, url }) { + if (url.startsWith('file://')) { + const { dispose } = await getSyncedHandler({ url }) + await dispose() + } + + const existing = await dispatchClient.backupRepository.get({ name }) + if (existing) { + assertRepositoryMatchesConfig(existing, url) + return existing + } + + const id = await dispatchClient.backupRepository.create(name, { url }) + const created = await dispatchClient.backupRepository.get({ id }) + if (!created) { + throw new Error(`Failed to retrieve created backup repository ${id}`) + } + tracker?.trackResource('backupRepository', id, { name }) + log.debug('Created backup repository', { name, id }) + return created +} + /** * Extracts an error message from a backup task's result or error fields. * diff --git a/@xen-orchestra/qa-test/utils/fleetUtils.js b/@xen-orchestra/qa-test/utils/fleetUtils.js new file mode 100644 index 0000000000..ae398b2508 --- /dev/null +++ b/@xen-orchestra/qa-test/utils/fleetUtils.js @@ -0,0 +1,44 @@ +import { createLogger } from '@xen-orchestra/log' +import assert from 'node:assert' +import { asyncEach } from '@vates/async-each' + +const log = createLogger('xo:qa-test:fleet') + +// Bounded by concurrency instead of one Promise.all over the whole fleet. +export async function cloneFleet({ + dispatchClient, + referenceVm, + count, + namePrefix, + tracker, + concurrency = 4, + bootTimeout = 120_000, + ipTimeout = 300_000, +}) { + assert(Number.isInteger(count) && count > 0, 'cloneFleet requires a positive integer count') + + const fleet = [] + + await asyncEach( + Array.from({ length: count }, (_, index) => index), + async index => { + const name = `${namePrefix}-${index}` + + const id = await dispatchClient.vm.clone(referenceVm.uuid, name, { + description: 'Load-test VM (churn fleet)', + fastClone: true, + }) + tracker?.trackResource('vm', id, { name, source: referenceVm.name_label }) + + await dispatchClient.vm.start(id) + await dispatchClient.vm.waitForPowerState(id, 'Running', bootTimeout) + const ip = await dispatchClient.vm.waitForGuestIp(id, ipTimeout) + + log.debug('Fleet VM ready', { name, id, ip }) + fleet.push({ id, ip }) + }, + { concurrency } + ) + + return fleet +} diff --git a/@xen-orchestra/qa-test/utils/index.js b/@xen-orchestra/qa-test/utils/index.js index 70007b9338..3c0eb15eaa 100644 --- a/@xen-orchestra/qa-test/utils/index.js +++ b/@xen-orchestra/qa-test/utils/index.js @@ -126,6 +126,23 @@ export function generateBackupJobName() { return `${BACKUP_JOB_NAME_PREFIX}${Date.now()}` } +/** + * Converts a non-empty list of resource IDs into a backupNg "simple pattern". + * + * Inverse of `extractIdsFromSimplePattern` (@xen-orchestra/backups): + * - one id: { id: 'a' } + * - several ids: { id: { __or: ['a', 'b'] } } + * + * Callers must guard against empty input; an empty list yields the degenerate + * pattern `{ id: { __or: [] } }`, which matches nothing. + * + * @param {ReadonlyArray} ids - Resource IDs (vms, remotes, srs, pools…) + * @returns {{ id: string | { __or: string[] } }} The simple pattern + */ +export function toSimplePattern(ids) { + return ids.length === 1 ? { id: ids[0] } : { id: { __or: ids } } +} + export async function waitUntil(conditionFn, interval = 1000, timeout = 15_000, options = {}) { const startTime = Date.now() let currentInterval = interval @@ -594,6 +611,55 @@ export function getBackupTransferredBytes(logEntry) { return findTransferSize(logEntry.tasks || []) } +// Unlike getBackupTransferredBytes (first match only), sums across every VM in the log. +export function sumBackupTransferredBytes(logEntry) { + let total = 0 + + function walk(tasks) { + for (const task of tasks ?? []) { + if (task.message === 'transfer' && task.result?.size !== undefined) { + total += task.result.size + } + if (task.tasks?.length > 0) { + walk(task.tasks) + } + } + } + + walk(logEntry?.tasks || []) + return total +} + +// Sums, across every VM in the log, snapshot duration and both clean-vm calls: the one before +// export (pre-run cleanup, normally a no-op) and the one inside export (post-transfer, where a +// merge triggered by exportRetention actually happens) — kept separate since only the latter +// tends to get expensive. +export function sumBackupPhaseDurations(logEntry) { + let snapshotDurationMs = 0 + let cleanVmBeforeDurationMs = 0 + let cleanVmAfterDurationMs = 0 + + const duration = task => (task.end ?? task.start) - task.start + + for (const vmTask of logEntry?.tasks ?? []) { + for (const task of vmTask.tasks ?? []) { + if (task.message === 'snapshot') { + snapshotDurationMs += duration(task) + } else if (task.message === 'clean-vm') { + cleanVmBeforeDurationMs += duration(task) + } else if (task.message === 'export') { + for (const exportTask of task.tasks ?? []) { + if (exportTask.message === 'clean-vm') { + cleanVmAfterDurationMs += duration(exportTask) + } + } + } + } + } + + return { snapshotDurationMs, cleanVmBeforeDurationMs, cleanVmAfterDurationMs } +} + /** * Asserts that incremental backup transferred fewer bytes or equal bytes than the baseline. * diff --git a/@xen-orchestra/qa-test/utils/vmChurn.js b/@xen-orchestra/qa-test/utils/vmChurn.js new file mode 100644 index 0000000000..ad84361b67 --- /dev/null +++ b/@xen-orchestra/qa-test/utils/vmChurn.js @@ -0,0 +1,55 @@ +import { createLogger } from '@xen-orchestra/log' +import assert from 'node:assert' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const log = createLogger('xo:qa-test:vm-churn') +const execFileAsync = promisify(execFile) + +// AES-CTR keystream from a small urandom seed: faster than /dev/urandom directly and, +// unlike /dev/zero, incompressible — so the backup pipeline sees a realistic transfer size. +// Host-key checking is disabled since short-lived clones commonly reuse DHCP addresses. +export async function churnVm({ + ip, + identityFile, + sizeMb, + user = 'root', + targetPath = '~/.qa-churn.bin', + timeout = 30_000, +}) { + assert(ip, 'churnVm requires the guest IP (e.g. from vm.waitForGuestIp)') + assert(Number.isFinite(sizeMb) && sizeMb > 0, 'churnVm requires a positive sizeMb') + + const remoteCommand = + `openssl enc -aes-256-ctr -pass pass:"$(od -An -tx1 -N32 /dev/urandom | tr -d ' \\n')" -nosalt /dev/null ` + + `| head -c ${sizeMb}M > ${targetPath} && sync` + + const args = [ + '-o', + 'StrictHostKeyChecking=no', + '-o', + 'UserKnownHostsFile=/dev/null', + '-o', + 'BatchMode=yes', + '-o', + 'ConnectTimeout=5', + // Omitted: falls back to ssh's own default identity files / ssh-agent. + ...(identityFile ? ['-i', identityFile] : []), + `${user}@${ip}`, + remoteCommand, + ] + + try { + await execFileAsync('ssh', args, { timeout }) + log.debug('Churned VM disk content', { ip, sizeMb }) + } catch (error) { + throw new Error(`Failed to churn ${sizeMb}MB on ${ip}`, { cause: error }) + } +} + +export function churnSizeMb(diskSizeBytes, percent) { + assert(Number.isFinite(diskSizeBytes) && diskSizeBytes > 0, 'churnSizeMb requires a positive diskSizeBytes') + assert(Number.isFinite(percent) && percent > 0, 'churnSizeMb requires a positive percent') + + return Math.round((diskSizeBytes * percent) / 100 / 1024 / 1024) +}