diff --git a/@vates/async-each/index.test.js b/@vates/async-each/index.test.js index 5caa5ba0b8..748a9d80cd 100644 --- a/@vates/async-each/index.test.js +++ b/@vates/async-each/index.test.js @@ -67,20 +67,17 @@ describe('asyncEach', () => { }) it('stops on first error when stopOnError is true', async () => { - const tracker = new assert.CallTracker() - const error = new Error() - const iteratee = tracker.calls((_, i) => { + const iteratee = spy((_, i) => { if (i === 1) { throw error } - }, 2) + }) assert.deepStrictEqual( await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true })), error ) - - tracker.verify() + assert.deepStrictEqual(iteratee.callCount, 2) }) it('rejects AggregateError when stopOnError is false', async () => { @@ -100,19 +97,16 @@ describe('asyncEach', () => { }) it('can be interrupted with an AbortSignal', async () => { - const tracker = new assert.CallTracker() - const ac = new AbortController() - const iteratee = tracker.calls((_, i) => { + const iteratee = spy((_, i) => { if (i === 1) { ac.abort() } - }, 2) + }) await assert.rejects(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal }), { message: 'asyncEach aborted', }) - - tracker.verify() + assert.deepStrictEqual(iteratee.callCount, 2) }) it('handle error on iterable', async () => { diff --git a/@xen-orchestra/disk-transform/src/utils/DebugConsumer.mts b/@xen-orchestra/disk-transform/src/utils/DebugConsumer.mts index 834a2cea80..51d9347a1a 100644 --- a/@xen-orchestra/disk-transform/src/utils/DebugConsumer.mts +++ b/@xen-orchestra/disk-transform/src/utils/DebugConsumer.mts @@ -13,7 +13,7 @@ export async function consume(disk: Disk, { blockDelay = 20, concurrency = 8 } = await asyncEach( disk.diskBlocks(), - async (block: DiskBlock) => { + async block => { consumed += block.data.length nbBlocks++ await new Promise(resolve => setTimeout(resolve, blockDelay)) diff --git a/@xen-orchestra/immutable-backups/_cleanXoCache.mjs b/@xen-orchestra/immutable-backups/_cleanXoCache.mjs deleted file mode 100644 index e2ba6de84d..0000000000 --- a/@xen-orchestra/immutable-backups/_cleanXoCache.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import fs from 'node:fs/promises' -import { dirname, join } from 'node:path' -import isBackupMetadata from './isBackupMetadata.mjs' - -export default async path => { - if (isBackupMetadata(path)) { - // snipe vm metadata cache to force XO to update it - await fs.unlink(join(dirname(path), 'cache.json.gz')) - } -} diff --git a/@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs b/@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs deleted file mode 100644 index a55bd13821..0000000000 --- a/@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs +++ /dev/null @@ -1,4 +0,0 @@ -import { dirname } from 'node:path' - -// check if we are handling file directly under a vhd directory ( bat, headr, footer,..) -export default path => dirname(path).endsWith('.vhd') diff --git a/@xen-orchestra/immutable-backups/_loadConfig.mjs b/@xen-orchestra/immutable-backups/_loadConfig.mjs deleted file mode 100644 index b8050505d1..0000000000 --- a/@xen-orchestra/immutable-backups/_loadConfig.mjs +++ /dev/null @@ -1,46 +0,0 @@ -import { load } from 'app-conf' -import { homedir } from 'os' -import { join } from 'node:path' -import ms from 'ms' - -const APP_NAME = 'xo-immutable-backups' -const APP_DIR = new URL('.', import.meta.url).pathname - -export default async function loadConfig() { - const config = await load(APP_NAME, { - appDir: APP_DIR, - ignoreUnknownFormats: true, - }) - if (config.remotes === undefined || config.remotes?.length < 1) { - throw new Error( - 'No remotes are configured in the config file, please add at least one [remotes.] with a root property pointing to the absolute path of the remote to watch' - ) - } - if (config.liftEvery) { - config.liftEvery = ms(config.liftEvery) - } - for (const [remoteId, { indexPath, immutabilityDuration, root }] of Object.entries(config.remotes)) { - if (!root) { - throw new Error( - `Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository ` - ) - } - if (!immutabilityDuration) { - throw new Error( - `Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be protected by immutability ` - ) - } - if (ms(immutabilityDuration) < ms('1d')) { - throw new Error( - `Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}` - ) - } - if (!indexPath) { - const basePath = indexPath ?? process.env.XDG_DATA_HOME ?? join(homedir(), '.local', 'share') - const immutabilityIndexPath = join(basePath, APP_NAME, remoteId) - config.remotes[remoteId].indexPath = immutabilityIndexPath - } - config.remotes[remoteId].immutabilityDuration = ms(immutabilityDuration) - } - return config -} diff --git a/@xen-orchestra/immutable-backups/directory.integ.mjs b/@xen-orchestra/immutable-backups/directory.integ.mjs deleted file mode 100644 index 56a720161a..0000000000 --- a/@xen-orchestra/immutable-backups/directory.integ.mjs +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, it } from 'node:test' -import assert from 'node:assert/strict' -import fs from 'node:fs/promises' -import path from 'node:path' -import { tmpdir } from 'node:os' -import * as Directory from './directory.mjs' -import { rimraf } from 'rimraf' - -describe('immutable-backups/file', async () => { - it('really lock a directory', async () => { - const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) - const dataDir = path.join(dir, 'data') - await fs.mkdir(dataDir) - const immutDir = path.join(dir, '.immutable') - const filePath = path.join(dataDir, 'test') - await fs.writeFile(filePath, 'data') - await Directory.makeImmutable(dataDir, immutDir) - assert.strictEqual(await Directory.isImmutable(dataDir), true) - await assert.rejects(() => fs.writeFile(filePath, 'data')) - await assert.rejects(() => fs.appendFile(filePath, 'data')) - await assert.rejects(() => fs.unlink(filePath)) - await assert.rejects(() => fs.rename(filePath, filePath + 'copy')) - await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data')) - await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy')) - await Directory.liftImmutability(dataDir, immutDir) - assert.strictEqual(await Directory.isImmutable(dataDir), false) - await fs.writeFile(filePath, 'data') - await fs.appendFile(filePath, 'data') - await fs.unlink(filePath) - await fs.rename(dataDir, dataDir + 'copy') - await rimraf(dir) - }) -}) diff --git a/@xen-orchestra/immutable-backups/directory.mjs b/@xen-orchestra/immutable-backups/directory.mjs deleted file mode 100644 index 761ee433d8..0000000000 --- a/@xen-orchestra/immutable-backups/directory.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import execa from 'execa' -import { unindexFile, indexFile } from './fileIndex.mjs' - -export async function makeImmutable(dirPath, immutabilityCachePath) { - if (immutabilityCachePath) { - await indexFile(dirPath, immutabilityCachePath) - } - await execa('chattr', ['+i', '-R', dirPath]) -} - -export async function liftImmutability(dirPath, immutabilityCachePath) { - if (immutabilityCachePath) { - await unindexFile(dirPath, immutabilityCachePath) - } - await execa('chattr', ['-i', '-R', dirPath]) -} - -export async function isImmutable(path) { - const { stdout } = await execa('lsattr', ['-d', path]) - return stdout[4] === 'i' -} diff --git a/@xen-orchestra/immutable-backups/file.mjs b/@xen-orchestra/immutable-backups/file.mjs deleted file mode 100644 index 6cc8f1ddda..0000000000 --- a/@xen-orchestra/immutable-backups/file.mjs +++ /dev/null @@ -1,24 +0,0 @@ -import execa from 'execa' -import { unindexFile, indexFile } from './fileIndex.mjs' - -// this work only on linux like systems -// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/ - -export async function makeImmutable(path, immutabilityCachePath) { - if (immutabilityCachePath) { - await indexFile(path, immutabilityCachePath) - } - await execa('chattr', ['+i', path]) -} - -export async function liftImmutability(filePath, immutabilityCachePath) { - if (immutabilityCachePath) { - await unindexFile(filePath, immutabilityCachePath) - } - await execa('chattr', ['-i', filePath]) -} - -export async function isImmutable(path) { - const { stdout } = await execa('lsattr', ['-d', path]) - return stdout[4] === 'i' -} diff --git a/@xen-orchestra/immutable-backups/fileIndex.integ.mjs b/@xen-orchestra/immutable-backups/fileIndex.integ.mjs deleted file mode 100644 index 844714bbcb..0000000000 --- a/@xen-orchestra/immutable-backups/fileIndex.integ.mjs +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, it } from 'node:test' -import assert from 'node:assert/strict' -import fs from 'node:fs/promises' -import path from 'node:path' -import * as FileIndex from './fileIndex.mjs' -import * as Directory from './directory.mjs' -import { tmpdir } from 'node:os' -import { rimraf } from 'rimraf' - -describe('immutable-backups/fileIndex', async () => { - it('index File changes', async () => { - const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) - const immutDir = path.join(dir, '.immutable') - const filePath = path.join(dir, 'test.ext') - - await fs.writeFile(filePath, 'data') - await FileIndex.indexFile(filePath, immutDir) - await fs.mkdir(path.join(immutDir, 'NOTADATE')) - await fs.writeFile(path.join(immutDir, 'NOTADATE.file'), 'content') - let nb = 0 - let index, target - for await ({ index, target } of FileIndex.listOlderTargets(immutDir, 0)) { - assert.strictEqual(true, false, 'Nothing should be eligible for deletion') - } - nb = 0 - for await ({ index, target } of FileIndex.listOlderTargets(immutDir, -24 * 60 * 60 * 1000)) { - assert.strictEqual(target, filePath) - await fs.unlink(index) - nb++ - } - assert.strictEqual(nb, 1) - await fs.rmdir(path.join(immutDir, 'NOTADATE')) - await fs.rm(path.join(immutDir, 'NOTADATE.file')) - for await ({ index, target } of FileIndex.listOlderTargets(immutDir, -24 * 60 * 60 * 1000)) { - // should remove the empty dir - assert.strictEqual(true, false, 'Nothing should have stayed here') - } - assert.strictEqual((await fs.readdir(immutDir)).length, 0) - await rimraf(dir) - }) - - it('fails correctly', async () => { - const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) - const immutDir = path.join(dir, '.immutable') - await fs.mkdir(immutDir) - const placeholderFile = path.join(dir, 'test.ext') - await fs.writeFile(placeholderFile, 'data') - await FileIndex.indexFile(placeholderFile, immutDir) - - const filePath = path.join(dir, 'test2.ext') - await fs.writeFile(filePath, 'data') - await FileIndex.indexFile(filePath, immutDir) - await assert.rejects(() => FileIndex.indexFile(filePath, immutDir), { code: 'EEXIST' }) - - await Directory.makeImmutable(immutDir) - await assert.rejects(() => FileIndex.unindexFile(filePath, immutDir), { code: 'EPERM' }) - await Directory.liftImmutability(immutDir) - await rimraf(dir) - }) - - it('handles bomb index files', async () => { - const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) - const immutDir = path.join(dir, '.immutable') - await fs.mkdir(immutDir) - const placeholderFile = path.join(dir, 'test.ext') - await fs.writeFile(placeholderFile, 'data') - await FileIndex.indexFile(placeholderFile, immutDir) - - const indexDayDir = path.join(immutDir, '1980,11-28') - await fs.mkdir(indexDayDir) - await fs.writeFile(path.join(indexDayDir, 'big'), Buffer.alloc(2 * 1024 * 1024)) - assert.rejects(async () => { - let index, target - for await ({ index, target } of FileIndex.listOlderTargets(immutDir, 0)) { - // should remove the empty dir - assert.strictEqual(true, false, `Nothing should have stayed here, got ${index} ${target}`) - } - }) - await rimraf(dir) - }) -}) diff --git a/@xen-orchestra/immutable-backups/fileIndex.mjs b/@xen-orchestra/immutable-backups/fileIndex.mjs deleted file mode 100644 index 1004e0d535..0000000000 --- a/@xen-orchestra/immutable-backups/fileIndex.mjs +++ /dev/null @@ -1,88 +0,0 @@ -import { join } from 'node:path' -import { createHash } from 'node:crypto' -import fs from 'node:fs/promises' -import { dirname } from 'path' -const MAX_INDEX_FILE_SIZE = 1024 * 1024 -function sha256(content) { - return createHash('sha256').update(content).digest('hex') -} - -function formatDate(date) { - return date.toISOString().split('T')[0] -} - -async function computeIndexFilePath(path, immutabilityIndexPath) { - const stat = await fs.stat(path) - const date = new Date(stat.birthtimeMs) - const day = formatDate(date) - const hash = sha256(path) - return join(immutabilityIndexPath, day, hash) -} - -export async function indexFile(path, immutabilityIndexPath) { - const indexFilePath = await computeIndexFilePath(path, immutabilityIndexPath) - try { - await fs.writeFile(indexFilePath, path, { flag: 'wx' }) - } catch (err) { - // missing dir: make it - if (err.code === 'ENOENT') { - await fs.mkdir(dirname(indexFilePath), { recursive: true }) - await fs.writeFile(indexFilePath, path) - } else { - throw err - } - } - return indexFilePath -} - -export async function unindexFile(path, immutabilityIndexPath) { - try { - const cacheFileName = await computeIndexFilePath(path, immutabilityIndexPath) - await fs.unlink(cacheFileName) - } catch (err) { - if (err.code !== 'ENOENT') { - throw err - } - } -} - -export async function* listOlderTargets(immutabilityCachePath, immutabilityDuration) { - // walk all dir by day until the limit day - const limitDate = new Date(Date.now() - immutabilityDuration) - - const limitDay = formatDate(limitDate) - const dir = await fs.opendir(immutabilityCachePath) - for await (const dirent of dir) { - if (dirent.isFile()) { - continue - } - // ensure we have a valid date - if (isNaN(new Date(dirent.name))) { - continue - } - // recent enough to be kept - if (dirent.name >= limitDay) { - continue - } - const subDirPath = join(immutabilityCachePath, dirent.name) - const subdir = await fs.opendir(subDirPath) - let nb = 0 - for await (const hashFileEntry of subdir) { - const entryFullPath = join(subDirPath, hashFileEntry.name) - const { size } = await fs.stat(entryFullPath) - if (size > MAX_INDEX_FILE_SIZE) { - throw new Error(`Index file at ${entryFullPath} is too big, ${size} bytes `) - } - const targetPath = await fs.readFile(entryFullPath, { encoding: 'utf8' }) - yield { - index: entryFullPath, - target: targetPath, - } - nb++ - } - // cleanup older folder - if (nb === 0) { - await fs.rmdir(subDirPath) - } - } -} diff --git a/@xen-orchestra/immutable-backups/isBackupMetadata.mjs b/@xen-orchestra/immutable-backups/isBackupMetadata.mjs deleted file mode 100644 index 47798a2acb..0000000000 --- a/@xen-orchestra/immutable-backups/isBackupMetadata.mjs +++ /dev/null @@ -1 +0,0 @@ -export default path => path.match(/xo-vm-backups\/[^/]+\/[^/]+\.json$/) diff --git a/@xen-orchestra/immutable-backups/liftProtection.mjs b/@xen-orchestra/immutable-backups/liftProtection.mjs deleted file mode 100755 index ba02956f9d..0000000000 --- a/@xen-orchestra/immutable-backups/liftProtection.mjs +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env node - -import * as Directory from './directory.mjs' -import { createLogger } from '@xen-orchestra/log' -import { listOlderTargets } from './fileIndex.mjs' -import cleanXoCache from './_cleanXoCache.mjs' -import loadConfig from './_loadConfig.mjs' - -const { info, warn } = createLogger('xen-orchestra:immutable-backups:liftProtection') - -async function liftRemoteImmutability(immutabilityCachePath, immutabilityDuration) { - for await (const { target } of listOlderTargets(immutabilityCachePath, immutabilityDuration)) { - await Directory.liftImmutability(target, immutabilityCachePath) - await cleanXoCache(target) - } -} - -async function liftImmutability(remotes) { - for (const [remoteId, { indexPath, immutabilityDuration }] of Object.entries(remotes)) { - liftRemoteImmutability(indexPath, immutabilityDuration).catch(err => - warn('error during watchRemote', { err, remoteId, indexPath, immutabilityDuration }) - ) - } -} - -const { liftEvery, remotes } = await loadConfig() - -if (liftEvery > 0) { - info('setup watcher for immutability lifting') - setInterval(async () => { - liftImmutability(remotes) - }, liftEvery) -} else { - liftImmutability(remotes) -} diff --git a/@xen-orchestra/immutable-backups/package.json b/@xen-orchestra/immutable-backups/package.json index 9df7c89932..33c1882d3a 100644 --- a/@xen-orchestra/immutable-backups/package.json +++ b/@xen-orchestra/immutable-backups/package.json @@ -13,30 +13,31 @@ "url": "https://vates.fr" }, "bin": { - "xo-immutable-remote": "./protectRemotes.mjs", - "xo-lift-remote-immutability": "./liftProtection.mjs" + "xo-immutable-remote": "./dist/protectRemotes.mjs" }, "license": "AGPL-3.0-or-later", "version": "1.0.30", "engines": { - "node": ">=14.0.0" + "node": ">=20.0.0" }, "dependencies": { "@vates/async-each": "^1.0.1", "@xen-orchestra/backups": "^0.69.4", "@xen-orchestra/log": "^0.7.1", "app-conf": "^3.0.0", - "chokidar": "^3.6.0", "execa": "^5.0.0", - "ms": "^2.1.3", - "vhd-lib": "^4.14.7" + "ms": "^2.1.3" }, "devDependencies": { + "@types/node": "^22.3.0", "rimraf": "^6.0.1", - "tap": "^18.6.1" + "tap": "^18.6.1", + "typescript": "~5.6" }, "scripts": { + "build": "tsc", + "clean": "rimraf dist/", "postversion": "npm publish --access public", - "test-integration": "tap *.integ.mjs" + "test-integration": "tsc && node --test dist/*.integ.mjs" } } diff --git a/@xen-orchestra/immutable-backups/protectRemotes.mjs b/@xen-orchestra/immutable-backups/protectRemotes.mjs deleted file mode 100755 index 87ebcbcdd4..0000000000 --- a/@xen-orchestra/immutable-backups/protectRemotes.mjs +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env node - -import fs from 'node:fs/promises' -import * as File from './file.mjs' -import * as Directory from './directory.mjs' -import assert from 'node:assert' -import { dirname, join, sep } from 'node:path' -import { createLogger } from '@xen-orchestra/log' -import chokidar from 'chokidar' -import { indexFile } from './fileIndex.mjs' -import cleanXoCache from './_cleanXoCache.mjs' -import loadConfig from './_loadConfig.mjs' -import isInVhdDirectory from './_isInVhdDirectory.mjs' -const { debug, info, warn } = createLogger('xen-orchestra:immutable-backups:remote') - -async function test(remotePath, indexPath) { - await fs.readdir(remotePath) - - const testPath = join(remotePath, '.test-immut') - // cleanup - try { - await File.liftImmutability(testPath, indexPath) - await fs.unlink(testPath) - } catch (err) {} - // can create , modify and delete a file - await fs.writeFile(testPath, `test immut ${new Date()}`) - await fs.writeFile(testPath, `test immut change 1 ${new Date()}`) - await fs.unlink(testPath) - - // cannot modify or delete an immutable file - await fs.writeFile(testPath, `test immut ${new Date()}`) - await File.makeImmutable(testPath, indexPath) - await assert.rejects(fs.writeFile(testPath, `test immut change 2 ${new Date()}`), { code: 'EPERM' }) - await assert.rejects(fs.unlink(testPath), { code: 'EPERM' }) - // can modify and delete a file after lifting immutability - await File.liftImmutability(testPath, indexPath) - - await fs.writeFile(testPath, `test immut change 3 ${new Date()}`) - await fs.unlink(testPath) -} -async function handleExistingFile(root, indexPath, path) { - try { - // a vhd block directory is completely immutable - if (isInVhdDirectory(path)) { - // this will trigger 3 times per vhd blocks - const dir = join(root, dirname(path)) - if (Directory.isImmutable(dir)) { - await indexFile(dir, indexPath) - } - } else { - // other files are immutable a file basis - const fullPath = join(root, path) - if (File.isImmutable(fullPath)) { - await indexFile(fullPath, indexPath) - } - } - } catch (error) { - if (error.code !== 'EEXIST') { - // there can be a symbolic link in the tree - warn('handleExistingFile', { error }) - } - } -} - -async function handleNewFile(root, indexPath, pendingVhds, path) { - // with awaitWriteFinish we have complete files here - // we can make them immutable - - if (isInVhdDirectory(path)) { - // watching a vhd block - // wait for header/footer and BAT before making this immutable recursively - const splitted = path.split(sep) - const vmUuid = splitted[1] - const vdiUuid = splitted[4] - const uniqPath = `${vmUuid}/${vdiUuid}` - const { existing } = pendingVhds.get(uniqPath) ?? {} - if (existing === undefined) { - pendingVhds.set(uniqPath, { existing: 1, lastModified: Date.now() }) - } else { - // already two of the key files,and we got the last one - if (existing === 2) { - await Directory.makeImmutable(join(root, dirname(path)), indexPath) - pendingVhds.delete(uniqPath) - } else { - // wait for the other - pendingVhds.set(uniqPath, { existing: existing + 1, lastModified: Date.now() }) - } - } - } else { - const fullFilePath = join(root, path) - await File.makeImmutable(fullFilePath, indexPath) - await cleanXoCache(fullFilePath) - } -} -export async function watchRemote(remoteId, { root, immutabilityDuration, rebuildIndexOnStart = false, indexPath }) { - // create index directory - await fs.mkdir(indexPath, { recursive: true }) - - // test if fs and index directories are well configured - await test(root, indexPath) - - // add duration and watch status in the metadata.json of the remote - const settingPath = join(root, 'immutability.json') - try { - // this file won't be made mutable by liftimmutability - await File.liftImmutability(settingPath) - } catch (error) { - // file may not exists, and it's not really a problem - info('lifting immutability on current settings', { error }) - } - await fs.writeFile( - settingPath, - JSON.stringify({ - since: Date.now(), - immutable: true, - duration: immutabilityDuration, - }) - ) - // no index path in makeImmutable(): the immutability won't be lifted - File.makeImmutable(settingPath) - - // we wait for footer/header AND BAT to be written before locking a vhd directory - // this map allow us to track the vhd with partial metadata - const pendingVhds = new Map() - // cleanup pending vhd map periodically - setInterval( - () => { - pendingVhds.forEach(({ lastModified, existing }, path) => { - if (Date.now() - lastModified > 60 * 60 * 1000) { - pendingVhds.delete(path) - warn(`vhd at ${path} is incomplete since ${lastModified}`, { existing, lastModified, path }) - } - }) - }, - 60 * 60 * 1000 - ) - - // watch the remote for any new VM metadata json file - const PATHS = [ - // xo-config-backups/scheduleId/date/metadata.json - 'xo-config-backups/*/*/data', - 'xo-config-backups/*/*/data.json', - 'xo-config-backups/*/*/metadata.json', - // xo-pool-metadata-backups/backupId/scheduleId/date/metadata.json - 'xo-pool-metadata-backups/*/*/*/metadata.json', - 'xo-pool-metadata-backups/*/*/*/data', - // xo-vm-backups// - 'xo-vm-backups/*/*.json', - 'xo-vm-backups/*/*.xva', - 'xo-vm-backups/*/*.xva.checksum', - // xo-vm-backups//vdis// - 'xo-vm-backups/*/vdis/*/*/*.vhd', // can be an alias or a vhd file - // for vhd directory : - 'xo-vm-backups/*/vdis/*/*/data/*.vhd/bat', - 'xo-vm-backups/*/vdis/*/*/data/*.vhd/header', - 'xo-vm-backups/*/vdis/*/*/data/*.vhd/footer', - ] - - let ready = false - const watcher = chokidar.watch(PATHS, { - ignored: [ - /(^|[/\\])\../, // ignore dotfiles - /\.lock$/, - ], - cwd: root, - recursive: false, // vhd directory can generate a lot of folder, don't let chokidar choke on this - ignoreInitial: !rebuildIndexOnStart, - depth: 7, - awaitWriteFinish: true, - }) - - // Add event listeners. - watcher - .on('add', async path => { - debug(`File ${path} has been added ${path.split('/').length}`) - if (ready) { - await handleNewFile(root, indexPath, pendingVhds, path) - } else { - await handleExistingFile(root, indexPath, path) - } - }) - .on('error', error => warn(`Watcher error: ${error}`)) - .on('ready', () => { - ready = true - info('Ready for changes') - }) -} - -const { remotes } = await loadConfig() - -for (const [remoteId, remote] of Object.entries(remotes)) { - watchRemote(remoteId, remote).catch(err => warn('error during watchRemote', { err, remoteId, remote })) -} diff --git a/@xen-orchestra/immutable-backups/src/_cleanXoCache.mts b/@xen-orchestra/immutable-backups/src/_cleanXoCache.mts new file mode 100644 index 0000000000..5a28ff01c1 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/_cleanXoCache.mts @@ -0,0 +1,16 @@ +import fs from 'node:fs/promises' +import { dirname, join } from 'node:path' +import isBackupMetadata from './isBackupMetadata.mjs' + +// If `path` is a VM backup metadata file, delete the adjacent `cache.json.gz` +// so that XO re-reads the updated metadata on next access. +export default async (path: string): Promise => { + if (isBackupMetadata(path)) { + // snipe vm metadata cache to force XO to update it + await fs.unlink(join(dirname(path), 'cache.json.gz')).catch(err => { + if (err.code !== 'ENOENT') { + throw err + } + }) + } +} diff --git a/@xen-orchestra/immutable-backups/src/_loadConfig.integ.mts b/@xen-orchestra/immutable-backups/src/_loadConfig.integ.mts new file mode 100644 index 0000000000..45b7cc7ae6 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/_loadConfig.integ.mts @@ -0,0 +1,100 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' + +import type { StringValue } from 'ms' + +import { parseConfig } from './_loadConfig.mjs' + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const VALID_REMOTE = { root: '/fake/root', immutabilityDuration: '30d' as StringValue } + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('parseConfig', () => { + it('throws when remotes key is absent', () => { + assert.throws(() => parseConfig({ liftEvery: '1h' } as any), /No remotes are configured/) + }) + + it('throws when a remote is missing the root property', () => { + assert.throws( + () => parseConfig({ liftEvery: '1h', remotes: { myRemote: { immutabilityDuration: '30d' } } }), + /don't have a root property/ + ) + }) + + it('throws when a remote is missing immutabilityDuration', () => { + assert.throws( + () => parseConfig({ liftEvery: '1h', remotes: { myRemote: { root: '/fake/root' } } }), + /don't have a immutabilityDuration/ + ) + }) + + it('throws when immutabilityDuration is shorter than 1 day', () => { + assert.throws( + () => + parseConfig({ liftEvery: '1h', remotes: { myRemote: { root: '/fake/root', immutabilityDuration: '23h' } } }), + /smaller than the minimum/ + ) + }) + + // --- liftEvery validation --- + + it('throws when liftEvery is a number', () => { + assert.throws( + () => parseConfig({ liftEvery: 3600 as any, remotes: { myRemote: VALID_REMOTE } }), + /must be a string for liftEvery/ + ) + }) + + it('throws when liftEvery is an unparsable string', () => { + assert.throws( + () => parseConfig({ liftEvery: 'notaduration' as any, remotes: { myRemote: VALID_REMOTE } }), + /is not a valid value for entry liftEvery/ + ) + }) + + // --- immutabilityDuration validation --- + + it('throws when immutabilityDuration is a number', () => { + assert.throws( + () => + parseConfig({ + liftEvery: '1h', + remotes: { myRemote: { root: '/fake/root', immutabilityDuration: 86400000 as any } }, + }), + /must be a string for remote myRemote/ + ) + }) + + it('throws when immutabilityDuration is an unparsable string', () => { + assert.throws( + () => + parseConfig({ + liftEvery: '1h', + remotes: { myRemote: { root: '/fake/root', immutabilityDuration: 'notaduration' as any } }, + }), + /is not a valid value for entry immutabilityDuration/ + ) + }) + + it('parses a valid duration string to milliseconds', () => { + const config = parseConfig({ + liftEvery: '1h', + remotes: { myRemote: { root: '/some/path', immutabilityDuration: '30d' } }, + }) + assert.strictEqual(config.remotes['myRemote'].immutabilityDuration, 30 * 24 * 60 * 60 * 1000) + }) + + it('parses liftEvery duration string to milliseconds', () => { + const config = parseConfig({ + liftEvery: '1h', + remotes: { myRemote: { root: '/some/path', immutabilityDuration: '7d' } }, + }) + assert.strictEqual(config.liftEvery, 60 * 60 * 1000) + }) +}) diff --git a/@xen-orchestra/immutable-backups/src/_loadConfig.mts b/@xen-orchestra/immutable-backups/src/_loadConfig.mts new file mode 100644 index 0000000000..29ad84a424 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/_loadConfig.mts @@ -0,0 +1,98 @@ +import { load } from 'app-conf' +import ms, { StringValue } from 'ms' + +// Configuration for a single watched remote. +// All duration fields are stored as milliseconds after config loading. +export interface RemoteConfig { + /** Absolute path to the root of the backup repository */ + root: string + /** Minimum duration in milliseconds that files must stay immutable */ + immutabilityDuration: number + /** Milliseconds to wait between consecutive size checks when polling for write completion (default: 100) */ + delayBetweenSizeCheck: number +} +export interface AppConfigInput { + liftEvery: StringValue + /** Map of remote ID to its resolved configuration */ + remotes: Record< + string, + Partial<{ root: string; immutabilityDuration: StringValue; delayBetweenSizeCheck: StringValue }> + > +} +export interface AppConfig { + /** Interval in milliseconds between immutability-lifting runs */ + liftEvery: number + /** Map of remote ID to its resolved configuration */ + remotes: Record +} + +const APP_NAME = 'xo-immutable-backups' +const APP_DIR = new URL('.', import.meta.url).pathname + +// Validate and transform a raw config object (as returned by app-conf) into a +// typed AppConfig. Duration strings are resolved to milliseconds. +// Exported so it can be unit-tested without touching the filesystem. +export function parseConfig(config: AppConfigInput): AppConfig { + const outputConfig: AppConfig = { + liftEvery: 0, + remotes: {}, + } + if (config.remotes === undefined || Object.keys(config.remotes).length < 1) { + throw new Error( + 'No remotes are configured in the config file, please add at least one [remotes.] with a root property pointing to the absolute path of the remote to watch' + ) + } + + if (typeof config.liftEvery !== 'string') { + throw new Error(`${config.liftEvery} must be a string for liftEvery`) + } + outputConfig.liftEvery = ms(config.liftEvery) + if (isNaN(outputConfig.liftEvery)) { + throw new Error(`${config.liftEvery} is not a valid value for entry liftEvery`) + } + for (const [remoteId, { immutabilityDuration, root, delayBetweenSizeCheck }] of Object.entries(config.remotes)) { + if (!root) { + throw new Error( + `Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository ` + ) + } + if (!immutabilityDuration) { + throw new Error( + `Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be protected by immutability ` + ) + } + if (typeof immutabilityDuration !== 'string') { + throw new Error(`${immutabilityDuration} is must be a string for remote ${remoteId}`) + } + if (isNaN(ms(immutabilityDuration))) { + throw new Error( + `${immutabilityDuration} is not a valid value for entry immutabilityDuration for remote ${remoteId}` + ) + } + if (ms(immutabilityDuration) < ms('1d')) { + throw new Error( + `Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}` + ) + } + + outputConfig.remotes[remoteId] = { + immutabilityDuration: ms(immutabilityDuration), + root, + delayBetweenSizeCheck: delayBetweenSizeCheck !== undefined ? ms(delayBetweenSizeCheck) : 100, + } + } + return outputConfig +} + +// Load the raw configuration from disk via app-conf. +export function loadRawConfig(): Promise { + return load(APP_NAME, { + appDir: APP_DIR, + ignoreUnknownFormats: true, + }) +} + +// Convenience default export: load from disk and parse in one call. +export default async function loadConfig(): Promise { + return parseConfig(await loadRawConfig()) +} diff --git a/@xen-orchestra/immutable-backups/src/_watcher.integ.mts b/@xen-orchestra/immutable-backups/src/_watcher.integ.mts new file mode 100644 index 0000000000..f561b22e5c --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/_watcher.integ.mts @@ -0,0 +1,219 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import { tmpdir } from 'node:os' +import { rimraf } from 'rimraf' +import execa from 'execa' + +import * as File from './file.mjs' +import { waitForWriteDone, watchVmDirectory } from './_watcher.mjs' + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function mkTmp(): Promise { + return fs.mkdtemp(path.join(tmpdir(), 'watcher-test-')) +} + +async function cleanupRoot(root: string): Promise { + try { + await execa('chattr', ['-i', '-R', root]) + } catch (_err) {} + await rimraf(root) +} + +// --------------------------------------------------------------------------- +// waitForWriteDone +// --------------------------------------------------------------------------- + +describe('waitForWriteDone', () => { + it('resolves immediately when file size is already stable', async () => { + const dir = await mkTmp() + try { + const file = path.join(dir, 'stable.bin') + await fs.writeFile(file, 'hello') + // Two polls must see the same size — give it 1 s timeout, should resolve fast + await waitForWriteDone(file, 1000, 100) + } finally { + await rimraf(dir) + } + }) + + it('waits for ENOENT then resolves once file appears with stable size', async () => { + const dir = await mkTmp() + try { + const file = path.join(dir, 'late.bin') + // Write the file 150 ms after the poll starts + setTimeout(async () => { + await fs.writeFile(file, 'data') + }, 150) + await waitForWriteDone(file, 2000, 100) + } finally { + await rimraf(dir) + } + }) + + it('does NOT resolve on an empty file (size === 0)', async () => { + const dir = await mkTmp() + try { + const file = path.join(dir, 'empty.bin') + await fs.writeFile(file, '') + // The function should time out because size is 0 — use a short timeout + await assert.rejects(waitForWriteDone(file, 400, 100), /Timeout/) + } finally { + await rimraf(dir) + } + }) + + it('rejects with Timeout when file never stabilises', async () => { + const dir = await mkTmp() + try { + const file = path.join(dir, 'growing.bin') + // Keep appending so size never stabilises + let stop = false + const writer = (async () => { + let i = 0 + while (!stop) { + await fs.appendFile(file, `chunk-${i++}\n`) + await new Promise(r => setTimeout(r, 80)) + } + })() + + await assert.rejects(waitForWriteDone(file, 500, 100), /Timeout/) + stop = true + await writer + } finally { + await rimraf(dir) + } + }) + + it('resolves once a growing file stops changing', async () => { + const dir = await mkTmp() + try { + const file = path.join(dir, 'finish.bin') + await fs.writeFile(file, 'chunk1') + // After 200 ms stop appending — function should resolve before 2 s + setTimeout(async () => { + await fs.appendFile(file, 'chunk2') + }, 50) + // No more writes after that; the size stabilises + await waitForWriteDone(file, 2000, 100) + } finally { + await rimraf(dir) + } + }) +}) + +// --------------------------------------------------------------------------- +// watchVmDirectory +// --------------------------------------------------------------------------- + +const VM_UUID = 'aaaaaaaa-0000-0000-0000-000000000001' +const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001' +const VDI_UUID = 'cccccccc-0000-0000-0000-000000000001' +const BACKUP_DATE = '20240115T120000Z' +const ONE_DAY_MS = 24 * 60 * 60 * 1000 + +async function waitFor(fn: () => Promise, timeout = 8000): Promise { + const deadline = Date.now() + timeout + while (Date.now() < deadline) { + try { + if (await fn()) return + } catch {} + await new Promise(resolve => setTimeout(resolve, 200)) + } + throw new Error('Timed out waiting for condition') +} + +describe('watchVmDirectory', () => { + it('ignores non-.json files', async () => { + const dir = await mkTmp() + const vmDir = path.join(dir, 'vm') + await fs.mkdir(vmDir, { recursive: true }) + + const errors: unknown[] = [] + const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 }) + try { + // Write a file that doesn't match *.json + const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`) + await fs.writeFile(xvaFile, 'fake xva') + // Give the watcher time to react + await new Promise(r => setTimeout(r, 500)) + // Must not be immutable — no .json was written + assert.strictEqual(await File.isImmutable(xvaFile), false, '.xva must stay mutable without .json') + assert.strictEqual(errors.length, 0, 'no errors expected') + } finally { + close() + await cleanupRoot(dir) + } + }) + + it('ignores json files that do not match datetime prefix (e.g. cache.json.gz)', async () => { + const dir = await mkTmp() + const vmDir = path.join(dir, 'vm') + await fs.mkdir(vmDir, { recursive: true }) + + const errors: unknown[] = [] + const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 }) + try { + const cacheFile = path.join(vmDir, 'cache.json') + await fs.writeFile(cacheFile, '{}') + await new Promise(r => setTimeout(r, 500)) + assert.strictEqual(await File.isImmutable(cacheFile), false, 'cache.json must stay mutable') + assert.strictEqual(errors.length, 0) + } finally { + close() + await cleanupRoot(dir) + } + }) + + it('deduplicates multiple fs events for the same datetime', async () => { + const dir = await mkTmp() + const vmDir = path.join(dir, 'vm') + await fs.mkdir(vmDir, { recursive: true }) + + let lockBackupCallCount = 0 + const errors: unknown[] = [] + + const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 }) + try { + const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`) + // Write the json file twice to trigger two fs events + await fs.writeFile(jsonFile, '{}') + await fs.writeFile(jsonFile, '{}') + + await waitFor(() => File.isImmutable(jsonFile)) + // The file should be immutable exactly once — lockBackup was called once + assert.strictEqual(await File.isImmutable(jsonFile), true) + assert.strictEqual(errors.length, 0) + } finally { + close() + await cleanupRoot(dir) + } + }) + + it('locks a flat VHD file when .json is written', async () => { + const dir = await mkTmp() + const vmDir = path.join(dir, 'vm') + const vdiDir = path.join(vmDir, 'vdis', JOB_UUID, VDI_UUID) + await fs.mkdir(vdiDir, { recursive: true }) + + const errors: unknown[] = [] + const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 }) + try { + const vhdFile = path.join(vdiDir, `${BACKUP_DATE}.vhd`) + const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`) + await fs.writeFile(vhdFile, 'fake vhd data') + await fs.writeFile(jsonFile, '{}') + + await waitFor(() => File.isImmutable(vhdFile)) + assert.strictEqual(await File.isImmutable(vhdFile), true, 'flat .vhd must be immutable') + assert.strictEqual(errors.length, 0) + } finally { + close() + await cleanupRoot(dir) + } + }) +}) diff --git a/@xen-orchestra/immutable-backups/src/_watcher.mts b/@xen-orchestra/immutable-backups/src/_watcher.mts new file mode 100644 index 0000000000..b63ea5ed69 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/_watcher.mts @@ -0,0 +1,386 @@ +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import { join } from 'node:path' +import * as Directory from './directory.mjs' +import { createLogger } from '@xen-orchestra/log' + +const { debug } = createLogger('xen-orchestra:immutable-backups:watcher') + +export interface WatchOptions { + /** Milliseconds after which a datetime is evicted from the deduplication set inside watchVmDirectory. */ + lockTimeout?: number + /** Milliseconds to wait between consecutive size checks when polling for write completion. */ + delayBetweenSizeCheck: number +} + +/** + * Matches the datetime prefix shared by all files belonging to a single backup run. + * + * Examples: + * "20231215T142030.json" → "20231215T142030" + * "20231215T142030.alias.vhd" → "20231215T142030" + * "cache.json.gz" → undefined (not a backup file) + */ +const DATETIME_RE = /^(\d{8}T\d{6}Z?)\./ + +function extractDatetime(filename: string): string | undefined { + return DATETIME_RE.exec(filename)?.[1] +} + +// Poll `path` with `fs.stat` until the file size stops changing, indicating the +// write is complete regardless of file format (plain or encrypted). +// The first stat is issued immediately; resolution requires a stable non-zero +// size across two consecutive polls spaced 100 ms apart. +// Rejects with an error if `timeout` ms elapse before stability is reached. +export async function waitForWriteDone(path: string, timeout: number, delayBetweenSizeCheck: number): Promise { + const deadline = Date.now() + timeout + let prevSize = -1 + + while (Date.now() < deadline) { + try { + const { size } = await fsp.stat(path) + if (size > 0 && size === prevSize) { + return // non-empty and size stable — write done + } + prevSize = size + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + // file not yet visible — keep waiting + } + await new Promise(resolve => setTimeout(resolve, delayBetweenSizeCheck)) + } + + throw new Error(`Timeout waiting for write to complete on ${path}`) +} + +// Lock every file belonging to the backup run identified by `datetime` inside `vmDir`. +// Called once the terminal `.json` file is fully written, which guarantees all other +// files for that run are already fully written. +// +// All candidates — flat files (json, xva, checksum, plain/alias VHDs) and VHD +// directories — are passed to a single `chattr +i -R` invocation. For regular +// files `-R` is a no-op; missing optional paths are silently ignored by +// Directory.makeImmutableBatch. +async function lockBackup(vmDir: string, datetime: string): Promise { + debug(`[watcher] lockBackup: vmDir="${vmDir}" datetime="${datetime}"`) + + const candidates: string[] = [ + join(vmDir, `${datetime}.json`), + join(vmDir, `${datetime}.xva`), + join(vmDir, `${datetime}.xva.checksum`), + ] + + const vdisDir = join(vmDir, 'vdis') + let jobIds: string[] = [] + try { + jobIds = await fsp.readdir(vdisDir) + } catch (err) { + if (err.code !== 'ENOENT') throw err + } + + await Promise.all( + jobIds.map(async jobId => { + let vdiIds: string[] + try { + vdiIds = await fsp.readdir(join(vdisDir, jobId)) + } catch { + // full backups won't have any vdi and it's ok + return + } + for (const vdiId of vdiIds) { + const vdiDir = join(vdisDir, jobId, vdiId) + candidates.push(join(vdiDir, `${datetime}.vhd`)) + candidates.push(join(vdiDir, `${datetime}.alias.vhd`)) + candidates.push(join(vdiDir, 'data', `${datetime}.vhd`)) + } + }) + ) + + await Directory.makeImmutableBatch(candidates) + + debug(`[watcher] lockBackup: done vmDir="${vmDir}" datetime="${datetime}"`) +} + +// Watch a single VM backup directory (`xo-vm-backups//`) for newly +// completed backups and lock all their files. +// +// Trigger: a `T.json` file appearing as a direct child whose +// size is stable (write complete). The `.json` is always written last, so its +// stability guarantees that every other file for that run is fully written. +// +// Returns a close function that stops the watcher. +export function watchVmDirectory( + vmDir: string, + onError: (err: unknown) => void, + { lockTimeout = 10 * 60 * 1000, delayBetweenSizeCheck }: WatchOptions +): () => void { + const watcher = fs.watch(vmDir) + + // Deduplicates lock attempts when multiple fs events fire for the same datetime. + // Entries are evicted after `lockTimeout` ms to keep the set bounded. + const lockedDatetimes = new Set() + + watcher.on('change', (_eventType: string, filename: string | Buffer | null) => { + if (filename == null) { + return + } + const name = String(filename) + debug(`[watcher] watchVmDirectory(${vmDir}): event filename="${name}"`) + if (!name.endsWith('.json')) { + return + } + const datetime = extractDatetime(name) + debug(`[watcher] watchVmDirectory: extractDatetime("${name}") → ${datetime}`) + if (datetime === undefined) { + debug(`[watcher] watchVmDirectory: ignoring "${name}" — does not match DATETIME_RE ${DATETIME_RE}`) + return // e.g. cache.json.gz + } + // ensure we handle event once per file + // clear up the remaingin data after a reasonnable tie + if (lockedDatetimes.has(datetime)) { + return + } + lockedDatetimes.add(datetime) + setTimeout(() => { + lockedDatetimes.delete(datetime) + }, lockTimeout).unref() + + const jsonPath = join(vmDir, name) + waitForWriteDone(jsonPath, lockTimeout, delayBetweenSizeCheck) + .then(() => { + debug(`[watcher] watchVmDirectory: locking backup datetime="${datetime}" in vmDir="${vmDir}"`) + return lockBackup(vmDir, datetime) + }) + .catch(err => { + const code = err.code + if (code === 'ENOENT') { + // file disappeared before write completed — next fs event will retry + debug(`[watcher] watchVmDirectory: file gone (ENOENT) — will retry on next event`) + return + } + onError(err) + }) + }) + + watcher.on('error', onError) + + return () => { + watcher.close() + watcher.removeAllListeners() + } +} + +// Watch a backup date directory (`T/`) for its terminal +// `metadata.json`. When its write is complete, all files in the directory are +// made immutable. +// +// Applies to both config backups and pool metadata backups: `metadata.json` +// is always written last. +// +// Returns a close function. +function watchBackupDateDirectory( + dateDir: string, + onError: (err: unknown) => void, + { lockTimeout = 10 * 60 * 1000, delayBetweenSizeCheck }: WatchOptions +): () => void { + const watcher = fs.watch(dateDir) + let locked = false + + function close() { + watcher.close() + watcher.removeAllListeners() + } + + watcher.on('change', (_eventType: string, filename: string | Buffer | null) => { + if (filename == null || String(filename) !== 'metadata.json') { + return + } + if (locked) { + return + } + const metadataPath = join(dateDir, 'metadata.json') + waitForWriteDone(metadataPath, lockTimeout, delayBetweenSizeCheck) + .then(() => { + if (locked) { + return + } + locked = true + return Directory.makeImmutable(dateDir).then(close) + }) + .catch(err => { + // we won't retry on error tot ensure we don't leave watcher dangling + close() + const code = err.code + if (code === 'ENOENT') { + return // maybe an incomplete write that has been purged + } + onError(err) + }) + }) + + watcher.on('error', err => { + close() + onError(err) + }) + return close +} + +// Watch `dir` for subdirectories, invoke `makeChildWatcher` for each one, +// and close the child watcher when the subdirectory is removed. +// +// `makeChildWatcher` may return a close function or a Promise that resolves to one. +async function watchSubdirectoriesWithChildren( + dir: string, + makeChildWatcher: (subdir: string) => (() => void) | Promise<() => void>, + onError: (err: unknown) => void +): Promise<() => void> { + const childClosers = new Map void>>() + + const closeWatcher = await watchSubdirectories( + dir, + { + onAdd: subdir => { + if (!childClosers.has(subdir)) { + childClosers.set( + subdir, + Promise.resolve(makeChildWatcher(subdir)).catch(err => { + onError(err) + return () => {} + }) + ) + } + }, + onRemove: subdir => { + const p = childClosers.get(subdir) + childClosers.delete(subdir) + p?.then(close => close()).catch(onError) + }, + }, + onError + ) + + return () => { + closeWatcher() + for (const p of childClosers.values()) { + p.then(close => close()).catch(onError) + } + childClosers.clear() + } +} + +// Watch `dir` for immediate subdirectory additions and deletions. Creates `dir` +// if it does not exist. Calls `onAdd` for each existing and new subdirectory, +// and `onRemove` when one disappears. +async function watchSubdirectories( + dir: string, + { onAdd, onRemove }: { onAdd: (subdir: string) => void; onRemove: (subdir: string) => void }, + onError: (err: unknown) => void +): Promise<() => void> { + await fsp.mkdir(dir, { recursive: true }) + + const watcher = fs.watch(dir) + + function close() { + watcher.close() + watcher.removeAllListeners() + } + + watcher.on('change', (_eventType: string, filename: string | Buffer | null) => { + if (filename == null) { + return + } + const subdir = join(dir, String(filename)) + debug(`[watcher] watchSubdirectories(${dir}): event filename="${String(filename)}"`) + fsp.stat(subdir).then( + stat => { + if (stat.isDirectory()) { + debug(`[watcher] watchSubdirectories: new dir detected "${subdir}"`) + onAdd(subdir) + } else { + debug(`[watcher] watchSubdirectories: "${subdir}" is not a directory (isFile=${stat.isFile()}) — ignored`) + } + }, + (err: NodeJS.ErrnoException) => { + if (err.code === 'ENOENT') { + onRemove(subdir) + return + } + onError(err) + } + ) + }) + + watcher.on('error', err => { + close() + onError(err) + }) + + const entries = await fsp.readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + if (entry.isDirectory()) { + onAdd(join(dir, entry.name)) + } + } + + return close +} + +// Watch a backup remote root and lock all completed backup files. +// +// Handles three backup types: +// - VM backups (`xo-vm-backups//`) — triggered by `T.json` +// - Config backups (`xo-config-backups//T/`) — triggered by `metadata.json` +// - Pool metadata (`xo-pool-metadata-backups///T/`) — triggered by `metadata.json` +// +// Returns a close function. +export async function watchRemote( + root: string, + onError: (err: unknown) => void, + options: WatchOptions +): Promise<() => void> { + const [closeVmWatcher, closeConfigWatcher, closePoolWatcher] = await Promise.all([ + // xo-vm-backups// + watchSubdirectoriesWithChildren( + join(root, 'xo-vm-backups'), + vmDir => watchVmDirectory(vmDir, onError, options), + onError + ), + + // xo-config-backups//T/ + watchSubdirectoriesWithChildren( + join(root, 'xo-config-backups'), + scheduleDir => + watchSubdirectoriesWithChildren( + scheduleDir, + dateDir => watchBackupDateDirectory(dateDir, onError, options), + onError + ), + onError + ), + + // xo-pool-metadata-backups///T/ + watchSubdirectoriesWithChildren( + join(root, 'xo-pool-metadata-backups'), + scheduleDir => + watchSubdirectoriesWithChildren( + scheduleDir, + poolDir => + watchSubdirectoriesWithChildren( + poolDir, + dateDir => watchBackupDateDirectory(dateDir, onError, options), + onError + ), + onError + ), + onError + ), + ]) + + return () => { + closeVmWatcher() + closeConfigWatcher() + closePoolWatcher() + } +} diff --git a/@xen-orchestra/immutable-backups/src/declarations.d.ts b/@xen-orchestra/immutable-backups/src/declarations.d.ts new file mode 100644 index 0000000000..e842836f6d --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/declarations.d.ts @@ -0,0 +1,4 @@ +declare module 'app-conf' { + function load(name: string, opts?: { appDir?: string; ignoreUnknownFormats?: boolean }): Promise + export { load } +} diff --git a/@xen-orchestra/immutable-backups/src/directory.integ.mts b/@xen-orchestra/immutable-backups/src/directory.integ.mts new file mode 100644 index 0000000000..d80cf589e8 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/directory.integ.mts @@ -0,0 +1,95 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import * as Directory from './directory.mjs' +import { tmpdir } from 'node:os' +import { rimraf } from 'rimraf' + +describe('immutable-backups/file', async () => { + it('really lock a directory', async () => { + const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) + try { + const dataDir = path.join(dir, 'data') + await fs.mkdir(dataDir) + const filePath = path.join(dataDir, 'test') + await fs.writeFile(filePath, 'data') + await Directory.makeImmutable(dataDir) + assert.strictEqual(await Directory.isImmutable(dataDir), true) + await assert.rejects(() => fs.writeFile(filePath, 'data')) + await assert.rejects(() => fs.appendFile(filePath, 'data')) + await assert.rejects(() => fs.unlink(filePath)) + await assert.rejects(() => fs.rename(filePath, filePath + 'copy')) + await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data')) + await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy')) + await Directory.liftImmutability(dataDir) + assert.strictEqual(await Directory.isImmutable(dataDir), false) + await fs.writeFile(filePath, 'data') + await fs.appendFile(filePath, 'data') + await fs.unlink(filePath) + await fs.rename(dataDir, dataDir + 'copy') + } finally { + await Directory.liftImmutabilityBatch([dir]).catch(() => {}) + await rimraf(dir) + } + }) + + it('makeImmutableBatch locks existing files and directories, ignores missing paths', async () => { + const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) + try { + const fileA = path.join(dir, 'a.txt') + const fileB = path.join(dir, 'b.txt') + const subDir = path.join(dir, 'sub') + const subFile = path.join(subDir, 'c.txt') + const missing = path.join(dir, 'does-not-exist') + + await fs.writeFile(fileA, 'aaa') + await fs.writeFile(fileB, 'bbb') + await fs.mkdir(subDir) + await fs.writeFile(subFile, 'ccc') + + // missing path must not throw + await assert.doesNotReject(() => Directory.makeImmutableBatch([fileA, fileB, subDir, missing])) + + assert.strictEqual(await Directory.isImmutable(fileA), true, 'fileA should be immutable') + assert.strictEqual(await Directory.isImmutable(fileB), true, 'fileB should be immutable') + assert.strictEqual(await Directory.isImmutable(subDir), true, 'subDir should be immutable') + assert.strictEqual(await Directory.isImmutable(subFile), true, 'subFile should be immutable (recursive)') + + await assert.rejects(() => fs.writeFile(fileA, 'tampered'), { code: 'EPERM' }) + await assert.rejects(() => fs.writeFile(subFile, 'tampered'), { code: 'EPERM' }) + } finally { + // lift before rimraf — rimraf cannot delete immutable files + await Directory.liftImmutabilityBatch([dir]).catch(() => {}) + await rimraf(dir) + } + }) + + it('liftImmutabilityBatch unlocks existing files and directories, ignores missing paths', async () => { + const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) + try { + const fileA = path.join(dir, 'a.txt') + const subDir = path.join(dir, 'sub') + const subFile = path.join(subDir, 'c.txt') + const missing = path.join(dir, 'does-not-exist') + + await fs.writeFile(fileA, 'aaa') + await fs.mkdir(subDir) + await fs.writeFile(subFile, 'ccc') + await Directory.makeImmutableBatch([fileA, subDir]) + + // missing path must not throw + await assert.doesNotReject(() => Directory.liftImmutabilityBatch([fileA, subDir, missing])) + + assert.strictEqual(await Directory.isImmutable(fileA), false, 'fileA should be mutable again') + assert.strictEqual(await Directory.isImmutable(subDir), false, 'subDir should be mutable again') + assert.strictEqual(await Directory.isImmutable(subFile), false, 'subFile should be mutable again') + + await fs.writeFile(fileA, 'updated') + await fs.writeFile(subFile, 'updated') + } finally { + await Directory.liftImmutabilityBatch([dir]).catch(() => {}) + await rimraf(dir) + } + }) +}) diff --git a/@xen-orchestra/immutable-backups/src/directory.mts b/@xen-orchestra/immutable-backups/src/directory.mts new file mode 100644 index 0000000000..d94ef14086 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/directory.mts @@ -0,0 +1,50 @@ +import execa from 'execa' + +// Recursively set the immutable (`+i`) attribute on a directory and all its contents. +export async function makeImmutable(dirPath: string): Promise { + await execa('chattr', ['+i', '-R', dirPath]) +} + +// chattr processes all paths even when some are missing (it does not abort on the first +// error), so every existing path is correctly lifted. Per-path "No such file or +// directory while trying to stat" messages are silently ignored; any other error +// (e.g. permission denied) causes the error to be re-thrown. +async function execChattrWithMissingFiles(args: string[]) { + try { + await execa('chattr', args) + } catch (err) { + const stderr: string = 'stderr' in err ? err.stderr : '' + const hasUnexpected = stderr + .split('\n') + .some(line => line.trim() !== '' && !line.includes('No such file or directory while trying to stat')) + if (hasUnexpected) { + throw err + } + } +} + +// Lock multiple paths (files and/or directories) with a single `chattr +i -R` invocation. +// For regular files `-R` is a no-op (chattr ignores it). Missing paths are silently +// ignored: chattr processes all remaining paths before exiting non-zero. +export async function makeImmutableBatch(paths: string[]): Promise { + if (paths.length === 0) return + await execChattrWithMissingFiles(['+i', '-R', ...paths]) +} + +// Recursively remove the immutable (`-i`) attribute from a directory and all its contents. +export async function liftImmutability(dirPath: string): Promise { + await execa('chattr', ['-i', '-R', dirPath]) +} + +// Lift immutability from multiple paths with a single `chattr -i -R` invocation. +// Works for both flat files and directories. +export async function liftImmutabilityBatch(paths: string[]): Promise { + if (paths.length === 0) return + await execChattrWithMissingFiles(['-i', '-R', ...paths]) +} + +// Returns whether the immutable (`i`) attribute is set on `path`. +export async function isImmutable(path: string): Promise { + const { stdout } = await execa('lsattr', ['-d', path]) + return stdout[4] === 'i' +} diff --git a/@xen-orchestra/immutable-backups/file.integ.mjs b/@xen-orchestra/immutable-backups/src/file.integ.mts similarity index 87% rename from @xen-orchestra/immutable-backups/file.integ.mjs rename to @xen-orchestra/immutable-backups/src/file.integ.mts index 5dac6d8df4..53e8dd3cf0 100644 --- a/@xen-orchestra/immutable-backups/file.integ.mjs +++ b/@xen-orchestra/immutable-backups/src/file.integ.mts @@ -9,17 +9,16 @@ import { rimraf } from 'rimraf' describe('immutable-backups/file', async () => { it('really lock a file', async () => { const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests')) - const immutDir = path.join(dir, '.immutable') const filePath = path.join(dir, 'test.ext') await fs.writeFile(filePath, 'data') assert.strictEqual(await File.isImmutable(filePath), false) - await File.makeImmutable(filePath, immutDir) + await File.makeImmutable(filePath) assert.strictEqual(await File.isImmutable(filePath), true) await assert.rejects(() => fs.writeFile(filePath, 'data')) await assert.rejects(() => fs.appendFile(filePath, 'data')) await assert.rejects(() => fs.unlink(filePath)) await assert.rejects(() => fs.rename(filePath, filePath + 'copy')) - await File.liftImmutability(filePath, immutDir) + await File.liftImmutability(filePath) assert.strictEqual(await File.isImmutable(filePath), false) await fs.writeFile(filePath, 'data') await fs.appendFile(filePath, 'data') diff --git a/@xen-orchestra/immutable-backups/src/file.mts b/@xen-orchestra/immutable-backups/src/file.mts new file mode 100644 index 0000000000..279a07e589 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/file.mts @@ -0,0 +1,20 @@ +import execa from 'execa' + +// this work only on linux like systems +// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/ + +// Set the immutable (`+i`) attribute on a single file. +export async function makeImmutable(path: string): Promise { + await execa('chattr', ['+i', path]) +} + +// Remove the immutable (`-i`) attribute from a single file. +export async function liftImmutability(filePath: string): Promise { + await execa('chattr', ['-i', filePath]) +} + +// Returns whether the immutable (`i`) attribute is set on `path`. +export async function isImmutable(path: string): Promise { + const { stdout } = await execa('lsattr', ['-d', path]) + return stdout[4] === 'i' +} diff --git a/@xen-orchestra/immutable-backups/src/isBackupMetadata.mts b/@xen-orchestra/immutable-backups/src/isBackupMetadata.mts new file mode 100644 index 0000000000..8fb32169b5 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/isBackupMetadata.mts @@ -0,0 +1,2 @@ +// Returns whether `path` points to a VM backup metadata JSON file. +export default (path: string): RegExpMatchArray | null => path.match(/xo-vm-backups\/[^/]+\/[^/]+\.json$/) diff --git a/@xen-orchestra/immutable-backups/src/liftProtection.mts b/@xen-orchestra/immutable-backups/src/liftProtection.mts new file mode 100644 index 0000000000..40569cc6a5 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/liftProtection.mts @@ -0,0 +1,183 @@ +import fsp from 'node:fs/promises' +import { join } from 'node:path' +import * as Directory from './directory.mjs' +import * as File from './file.mjs' +import { createLogger } from '@xen-orchestra/log' +import { asyncEach } from '@vates/async-each' +import cleanXoCache from './_cleanXoCache.mjs' +import { RemoteConfig } from './_loadConfig.mjs' + +const { warn } = createLogger('xen-orchestra:immutable-backups:liftProtection') + +/** + * Matches the datetime prefix shared by all files belonging to a single VM backup run. + * + * Examples: + * "20231215T142030.json" → "20231215T142030" + * "20231215T142030Z.alias.vhd" → "20231215T142030Z" + * "cache.json.gz" → undefined (not a backup file) + */ +const DATETIME_RE = /^(\d{8}T\d{6}Z?)\./ + +function extractDatetime(filename: string): string | undefined { + return DATETIME_RE.exec(filename)?.[1] +} + +// Returns the absolute paths of all immediate subdirectories of `dir`. +// Returns [] if `dir` does not exist. +async function listDirs(dir: string): Promise { + try { + const entries = await fsp.readdir(dir, { withFileTypes: true }) + return entries.filter(e => e.isDirectory()).map(e => join(dir, e.name)) + } catch (err) { + if (err.code === 'ENOENT') return [] + throw err + } +} + +// Lift immutability from every file and subdirectory inside a date-stamped +// backup directory (config backup or pool-metadata backup). +async function liftDirBackup(dateDir: string): Promise { + const entries = await fsp.readdir(dateDir, { withFileTypes: true }) + const paths = entries.map(entry => join(dateDir, entry.name)) + await Directory.liftImmutabilityBatch(paths) +} + +// Walk `xo-vm-backups//.json` files and lift immutability on +// any VM backup run whose metadata mtime is older than `immutabilityDuration`. +// Per vmDir: vdis is read once, all expired datetimes are batched into a single +// liftImmutabilityBatch call, and cleanXoCache is called once. +async function liftExpiredVmBackups(root: string, immutabilityDuration: number): Promise { + const threshold = Date.now() - immutabilityDuration + await asyncEach(await listDirs(join(root, 'xo-vm-backups')), async vmDir => { + // 1. Find all expired datetimes in this vmDir. + const entries = await fsp.readdir(vmDir, { withFileTypes: true }).catch(() => []) + const expiredDatetimes: string[] = [] + let firstExpiredJsonPath: string | undefined + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue + const datetime = extractDatetime(entry.name) + if (datetime === undefined) continue // e.g. cache.json.gz + const jsonPath = join(vmDir, entry.name) + try { + if (!(await File.isImmutable(jsonPath))) continue + const { mtimeMs } = await fsp.stat(jsonPath) + if (mtimeMs > threshold) continue + expiredDatetimes.push(datetime) + firstExpiredJsonPath ??= jsonPath + } catch (err) { + warn('error checking VM backup expiry', { err, jsonPath }) + } + } + + if (expiredDatetimes.length === 0) return + + // 2. Read the vdis tree once for this vmDir. + const vdisDir = join(vmDir, 'vdis') + let jobIds: string[] = [] + try { + jobIds = await fsp.readdir(vdisDir) + } catch (err) { + if (err.code !== 'ENOENT') throw err + } + const vdiDirs: string[] = [] + await asyncEach(jobIds, async jobId => { + let vdiIds: string[] + try { + vdiIds = await fsp.readdir(join(vdisDir, jobId)) + } catch { + return + } + for (const vdiId of vdiIds) { + vdiDirs.push(join(vdisDir, jobId, vdiId)) + } + }) + + // 3. Build all candidates for all expired datetimes in one pass. + const candidates: string[] = [] + for (const datetime of expiredDatetimes) { + candidates.push( + join(vmDir, `${datetime}.json`), + join(vmDir, `${datetime}.xva`), + join(vmDir, `${datetime}.xva.checksum`) + ) + for (const vdiDir of vdiDirs) { + candidates.push( + join(vdiDir, `${datetime}.vhd`), + join(vdiDir, `${datetime}.alias.vhd`), + join(vdiDir, 'data', `${datetime}.vhd`) + ) + } + } + + // 4. Single batch lift + single cache invalidation for this vmDir. + try { + await Directory.liftImmutabilityBatch(candidates) + await cleanXoCache(firstExpiredJsonPath!) + } catch (err) { + warn('error lifting VM backup immutability', { err, vmDir }) + } + }) +} + +// Walk `xo-config-backups///metadata.json` files and +// lift immutability on any backup directory whose metadata mtime is expired. +async function liftExpiredConfigBackups(root: string, immutabilityDuration: number): Promise { + const threshold = Date.now() - immutabilityDuration + await asyncEach(await listDirs(join(root, 'xo-config-backups')), async scheduleDir => { + for (const dateDir of await listDirs(scheduleDir)) { + const metadataPath = join(dateDir, 'metadata.json') + try { + if (!(await File.isImmutable(metadataPath))) continue + const { mtimeMs } = await fsp.stat(metadataPath) + if (mtimeMs > threshold) continue + await liftDirBackup(dateDir) + } catch (err) { + const code = err.code + if (code !== 'ENOENT') warn('error lifting config backup immutability', { err, metadataPath }) + } + } + }) +} + +// Walk `xo-pool-metadata-backups////metadata.json` +// files and lift immutability on any backup directory whose metadata mtime is expired. +async function liftExpiredPoolBackups(root: string, immutabilityDuration: number): Promise { + const threshold = Date.now() - immutabilityDuration + await asyncEach(await listDirs(join(root, 'xo-pool-metadata-backups')), async scheduleDir => { + for (const poolDir of await listDirs(scheduleDir)) { + for (const dateDir of await listDirs(poolDir)) { + const metadataPath = join(dateDir, 'metadata.json') + try { + if (!(await File.isImmutable(metadataPath))) continue + const { mtimeMs } = await fsp.stat(metadataPath) + if (mtimeMs > threshold) continue + await liftDirBackup(dateDir) + } catch (err) { + const code = err.code + if (code !== 'ENOENT') warn('error lifting pool metadata backup immutability', { err, metadataPath }) + } + } + } + }) +} + +// Scan the filesystem for expired immutable backups under `root` and lift their +// immutability. No index is required — the backup tree is walked directly. +export async function liftRemoteImmutability(root: string, immutabilityDuration: number): Promise { + await Promise.all([ + liftExpiredVmBackups(root, immutabilityDuration), + liftExpiredConfigBackups(root, immutabilityDuration), + liftExpiredPoolBackups(root, immutabilityDuration), + ]) +} + +// Lift immutability on all expired backups across every configured remote. +export async function liftImmutability(remotes: Record): Promise { + for (const [remoteId, { root, immutabilityDuration }] of Object.entries(remotes)) { + await liftRemoteImmutability(root, immutabilityDuration).catch(err => + warn('error during liftRemoteImmutability', { err, remoteId, root, immutabilityDuration }) + ) + } +} diff --git a/@xen-orchestra/immutable-backups/src/protectRemotes.mts b/@xen-orchestra/immutable-backups/src/protectRemotes.mts new file mode 100644 index 0000000000..b96b6b2a26 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/protectRemotes.mts @@ -0,0 +1,18 @@ +#!/usr/bin/env node + +import { createLogger } from '@xen-orchestra/log' +import loadConfig from './_loadConfig.mjs' +import { watchRemote } from './remote.mjs' +import { liftImmutability } from './liftProtection.mjs' + +const { info, warn } = createLogger('xen-orchestra:immutable-backups:remote') + +const { liftEvery, remotes } = await loadConfig() +for (const [remoteId, remote] of Object.entries(remotes)) { + watchRemote(remoteId, remote).catch(err => warn('error during watchRemote', { err, remoteId, remote })) +} + +info('setup watcher for immutability lifting') +setInterval(async () => { + await liftImmutability(remotes).catch(error => warn('error while lifting immutability', error)) +}, liftEvery) diff --git a/@xen-orchestra/immutable-backups/src/remote.integ.mts b/@xen-orchestra/immutable-backups/src/remote.integ.mts new file mode 100644 index 0000000000..a0972bc277 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/remote.integ.mts @@ -0,0 +1,241 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import { tmpdir } from 'node:os' +import { rimraf } from 'rimraf' +import execa from 'execa' + +import * as File from './file.mjs' +import * as Directory from './directory.mjs' +import { watchRemote } from './remote.mjs' + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const ONE_DAY_MS = 24 * 60 * 60 * 1000 + +// Fake UUIDs used across all tests +const VM_UUID = 'aaaaaaaa-0000-0000-0000-000000000001' +const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001' +const VDI_UUID = 'cccccccc-0000-0000-0000-000000000001' +const SCHEDULE_UUID = 'dddddddd-0000-0000-0000-000000000001' +const BACKUP_DATE = '20240115T120000Z' + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Poll `fn` every 200 ms until it returns a truthy value or `timeout` ms +// have elapsed. Throws if the deadline is reached. +async function waitFor(fn: () => Promise, timeout = 8000): Promise { + const deadline = Date.now() + timeout + while (Date.now() < deadline) { + try { + if (await fn()) return + } catch {} + await new Promise(resolve => setTimeout(resolve, 200)) + } + throw new Error('Timed out waiting for condition') +} + +// Recursively lift all immutable flags inside `root`, then delete the tree. +// Must be run as root since chattr requires elevated privileges. +async function cleanupRoot(root: string): Promise { + try { + // -i removes the immutable flag; -R recurses into every file and directory. + await execa('chattr', ['-i', '-R', root]) + } catch (_err) {} + await rimraf(root) +} + +// Start a watcher on a fresh temp directory and return both the root path and +// the index path so the caller can create files and assert on them. +// +// `preDirs` is a list of relative paths that will be created inside `root` +// **before** starting the watcher. The watcher must see these directories at +// startup so it can detect new files written into them later. +async function makeRemote(preDirs: string[] = []): Promise<{ root: string; close: () => void }> { + const root = await fs.mkdtemp(path.join(tmpdir(), 'immut-test-')) + // Pre-create directories so the watcher has them in its watch list at startup. + for (const relDir of preDirs) { + await fs.mkdir(path.join(root, relDir), { recursive: true }) + } + const { close } = await watchRemote('test', { root, immutabilityDuration: ONE_DAY_MS, delayBetweenSizeCheck: 100 }) + // Give the watcher a moment to complete its initial scan and become ready before + // the caller starts writing files. + await new Promise(resolve => setTimeout(resolve, 500)) + return { root, close } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('protectRemotes/watchRemote', async () => { + it('makes VM backup flat files (.json / .xva / .xva.checksum) immutable', async () => { + const vmRelDir = path.join('xo-vm-backups', VM_UUID) + const { root, close } = await makeRemote([vmRelDir]) + try { + const vmDir = path.join(root, vmRelDir) + + const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`) + const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`) + const checksumFile = path.join(vmDir, `${BACKUP_DATE}.xva.checksum`) + // This file does NOT match any watched glob — it must stay mutable. + const ignoredFile = path.join(vmDir, 'not-a-backup.txt') + + // xva and checksum must exist before json — json is the terminal signal + await fs.writeFile(xvaFile, 'fake xva data') + await fs.writeFile(checksumFile, 'abc123') + await fs.writeFile(ignoredFile, 'should stay mutable') + await fs.writeFile(jsonFile, '{}') + + await waitFor(() => File.isImmutable(jsonFile)) + await waitFor(() => File.isImmutable(xvaFile)) + await waitFor(() => File.isImmutable(checksumFile)) + + assert.strictEqual(await File.isImmutable(jsonFile), true, '.json should be immutable') + assert.strictEqual(await File.isImmutable(xvaFile), true, '.xva should be immutable') + assert.strictEqual(await File.isImmutable(checksumFile), true, '.xva.checksum should be immutable') + + // Confirm writes and deletes are actually blocked. + await assert.rejects(fs.writeFile(jsonFile, 'tampered'), { code: 'EPERM' }) + await assert.rejects(fs.unlink(xvaFile), { code: 'EPERM' }) + + // Files outside the watched patterns must remain mutable. + assert.strictEqual(await File.isImmutable(ignoredFile), false, 'non-backup file must stay mutable') + } finally { + close() + await cleanupRoot(root) + } + }) + + it('makes a VHD directory immutable once the json is written', async () => { + const vmRelDir = path.join('xo-vm-backups', VM_UUID) + const dataRelDir = path.join(vmRelDir, 'vdis', JOB_UUID, VDI_UUID, 'data') + const { root, close } = await makeRemote([dataRelDir]) + try { + // xo-vm-backups//vdis///data/.vhd/ + const vmDir = path.join(root, vmRelDir) + const vhdDir = path.join(root, dataRelDir, `${BACKUP_DATE}.vhd`) + await fs.mkdir(vhdDir, { recursive: true }) + + const bat = path.join(vhdDir, 'bat') + const header = path.join(vhdDir, 'header') + const footer = path.join(vhdDir, 'footer') + // Subdirectory inside the VHD dir that is NOT a watched key file. + const blocks = path.join(vhdDir, 'blocks') + await fs.mkdir(blocks) + + await fs.writeFile(bat, 'bat data') + await fs.writeFile(header, 'header data') + await fs.writeFile(footer, 'footer data') + + // json written last — triggers lockBackup which locks the VHD directory + await fs.writeFile(path.join(vmDir, `${BACKUP_DATE}.json`), '{}') + + // The whole directory (and its contents) must become immutable. + await waitFor(() => Directory.isImmutable(vhdDir)) + + assert.strictEqual(await Directory.isImmutable(vhdDir), true, 'vhd dir should be immutable') + assert.strictEqual(await File.isImmutable(bat), true, 'bat should be immutable') + assert.strictEqual(await File.isImmutable(header), true, 'header should be immutable') + assert.strictEqual(await File.isImmutable(footer), true, 'footer should be immutable') + + // Confirm the lock is real. + await assert.rejects(fs.writeFile(bat, 'tampered'), { code: 'EPERM' }) + await assert.rejects(fs.mkdir(path.join(vhdDir, 'new-subdir')), { code: 'EPERM' }) + } finally { + close() + await cleanupRoot(root) + } + }) + + it('does NOT lock the VHD directory when the metadata are not written', async () => { + const dataRelDir = path.join('xo-vm-backups', VM_UUID, 'vdis', JOB_UUID, VDI_UUID, 'data') + const { root, close } = await makeRemote([dataRelDir]) + try { + const vhdDir = path.join(root, dataRelDir, `${BACKUP_DATE}.vhd`) + await fs.mkdir(vhdDir, { recursive: true }) + + const bat = path.join(vhdDir, 'bat') + const header = path.join(vhdDir, 'header') + // footer is intentionally omitted. + + await fs.writeFile(bat, 'bat data') + await fs.writeFile(header, 'header data') + + // Wait longer than awaitWriteFinish (2 s) + processing to be certain + // the watcher has handled both events without locking anything. + await new Promise(resolve => setTimeout(resolve, 5000)) + + assert.strictEqual(await Directory.isImmutable(vhdDir), false, 'dir must stay mutable') + assert.strictEqual(await File.isImmutable(bat), false, 'bat must stay mutable') + assert.strictEqual(await File.isImmutable(header), false, 'header must stay mutable') + } finally { + close() + await cleanupRoot(root) + } + }) + + it('makes config-backup files immutable', async () => { + const scheduleRelDir = path.join('xo-config-backups', SCHEDULE_UUID) + const { root, close } = await makeRemote([scheduleRelDir]) + try { + // xo-config-backups/// + const configDir = path.join(root, scheduleRelDir, BACKUP_DATE) + await fs.mkdir(configDir, { recursive: true }) + + const dataFile = path.join(configDir, 'data') + const dataJsonFile = path.join(configDir, 'data.json') + const metadataFile = path.join(configDir, 'metadata.json') + + await fs.writeFile(dataFile, 'config backup data') + await fs.writeFile(dataJsonFile, '{}') + await fs.writeFile(metadataFile, '{}') + + await waitFor(() => File.isImmutable(dataFile)) + await waitFor(() => File.isImmutable(dataJsonFile)) + await waitFor(() => File.isImmutable(metadataFile)) + + assert.strictEqual(await File.isImmutable(dataFile), true, 'data should be immutable') + assert.strictEqual(await File.isImmutable(dataJsonFile), true, 'data.json should be immutable') + assert.strictEqual(await File.isImmutable(metadataFile), true, 'metadata.json should be immutable') + } finally { + close() + await cleanupRoot(root) + } + }) + + it('detects a new VM directory created after the watcher starts', async () => { + // Start with NO pre-existing VM directories. + // The watcher must dynamically pick up xo-vm-backups// created + // after it is already running — this is the core of the cascading design. + const { root, close } = await makeRemote([]) + try { + const newVmUuid = 'eeeeeeee-0000-0000-0000-000000000099' + const vmDir = path.join(root, 'xo-vm-backups', newVmUuid) + + // Create the VM directory and write backup files AFTER the watcher is running. + await fs.mkdir(vmDir, { recursive: true }) + + // xva must be written before json (json is the terminal signal). + const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`) + const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`) + await fs.writeFile(xvaFile, 'fake xva data') + await fs.writeFile(jsonFile, '{}') + + await waitFor(() => File.isImmutable(jsonFile)) + await waitFor(() => File.isImmutable(xvaFile)) + + assert.strictEqual(await File.isImmutable(jsonFile), true, '.json in new VM dir should be immutable') + assert.strictEqual(await File.isImmutable(xvaFile), true, '.xva in new VM dir should be immutable') + await assert.rejects(fs.writeFile(jsonFile, 'tampered'), { code: 'EPERM' }) + } finally { + close() + await cleanupRoot(root) + } + }) +}) diff --git a/@xen-orchestra/immutable-backups/src/remote.load.mts b/@xen-orchestra/immutable-backups/src/remote.load.mts new file mode 100644 index 0000000000..fc8e314bec --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/remote.load.mts @@ -0,0 +1,461 @@ +#!/usr/bin/env node +// +// Load test: 5000 VMs × 4 flat VHDs. +// +// Phase A — live watcher: +// Write backups for all VMs while the watcher is running and measure how +// long it takes for every backup to become immutable. +// +// Phase B — lift: +// Lift immutability on all locked files and measure the duration. +// +// Must be run as root (chattr requires elevated privileges). +// Usage: node @xen-orchestra/immutable-backups/remote.load.mjs + +import fs from 'node:fs/promises' +import path from 'node:path' +import { tmpdir } from 'node:os' +import { rimraf } from 'rimraf' +import execa from 'execa' + +import { asyncEach } from '@vates/async-each' +import * as File from './file.mjs' +import { watchRemote } from './remote.mjs' +import { liftRemoteImmutability } from './liftProtection.mjs' + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +const VM_COUNT = 5000 +const DISKS_PER_VM = 4 +const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001' + +/** Datetime used for the live backups (Phase B). */ +const BACKUP_DATE = '20240115T120000Z' + +const ONE_DAY_MS = 24 * 60 * 60 * 1000 + +/** Max concurrent backup file writes. */ +const WRITE_CONCURRENCY = 100 + +/** Max files per lsattr call (avoids ARG_MAX). */ +const LSATTR_BATCH = 500 + +/** Max time to wait for all backups to be locked (ms). */ +const LOCK_TIMEOUT_MS = 120_000 + +// Each VM backup: 1 .json + DISKS_PER_VM plain .vhd files. +const FILES_PER_VM = 1 + DISKS_PER_VM +const EXPECTED_TOTAL = VM_COUNT * FILES_PER_VM + +/** + * Indices into `vms[]` whose json→immutable latency is tracked individually + * to show per-backup timing under full concurrent load. + */ +const PROBE_VM_INDICES = [ + 0, + Math.floor(VM_COUNT / 4), + Math.floor(VM_COUNT / 2), + Math.floor((VM_COUNT * 3) / 4), + VM_COUNT - 1, +] + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeUuid(i: number): string { + return `${String(i).padStart(8, '0')}-0000-0000-0000-000000000000` +} + +async function cleanupRoot(root: string): Promise { + try { + await execa('chattr', ['-i', '-R', root]) + } catch {} + await rimraf(root) +} + +// Spin-polls File.isImmutable every 10 ms until the file is immutable or +// `timeoutMs` elapses. Returns the wall-clock timestamp (ms) when locked, +// or null on timeout. +async function waitUntilImmutable(filePath: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (await File.isImmutable(filePath).catch(() => false)) return Date.now() + await new Promise(r => setTimeout(r, 10)) + } + return null +} + +// Count how many `.json` files across all VM dirs are currently immutable. +// Uses batched lsattr calls to avoid hitting ARG_MAX. +async function countImmutableJsonFiles(root: string, date: string, vmUuids: string[]): Promise { + let count = 0 + for (let i = 0; i < vmUuids.length; i += LSATTR_BATCH) { + const batch = vmUuids.slice(i, i + LSATTR_BATCH).map(uuid => path.join(root, 'xo-vm-backups', uuid, `${date}.json`)) + try { + const { stdout } = await execa('lsattr', ['-d', ...batch]) + count += stdout.split('\n').filter(line => line.length > 4 && line[4] === 'i').length + } catch (err) { + // lsattr exits non-zero when some files don't exist yet — parse stdout anyway + const out = (err as any).stdout as string | undefined + if (out) count += out.split('\n').filter((line: string) => line.length > 4 && line[4] === 'i').length + } + } + return count +} + +// Poll `countImmutableJsonFiles` until it reaches `target` or `LOCK_TIMEOUT_MS` +// elapses. Logs progress on every change. +async function waitForImmutableCount( + root: string, + date: string, + vmUuids: string[], + target: number, + label: string +): Promise<{ elapsed: number; reached: boolean }> { + const tStart = Date.now() + let lastCount = await countImmutableJsonFiles(root, date, vmUuids) + console.log(` [${label}] starting at ${lastCount}/${target}`) + + const deadline = Date.now() + LOCK_TIMEOUT_MS + while (Date.now() < deadline) { + sampleMemory() + const count = await countImmutableJsonFiles(root, date, vmUuids) + if (count !== lastCount) { + console.log( + ` [${label}] immutable json files: ${count}/${target}` + + ` | ${Date.now() - tStart} ms elapsed` + + ` | peak RSS ${mb(peakRss)}` + ) + lastCount = count + } + if (count >= target) { + return { elapsed: Date.now() - tStart, reached: true } + } + await new Promise(resolve => setTimeout(resolve, 500)) + } + return { elapsed: Date.now() - tStart, reached: false } +} + +// --------------------------------------------------------------------------- +// Raw cost benchmark +// --------------------------------------------------------------------------- + +function average(arr: number[]): number { + return arr.reduce((a, b) => a + b, 0) / arr.length +} + +// Measures the cost of the individual operations that `lockBackup` performs, +// with no watcher running and no concurrent load, so results reflect pure +// operation overhead with no queue contention. +// +// Reports: +// 1. chattr +i — single subprocess spawn +// 2. 5 concurrent chattr +i — one realistic backup (1 json + 4 vhds in parallel) +async function measureRawCosts(scratchDir: string): Promise { + console.log('=== Baseline: raw lockBackup operation costs (no watcher, no load) ===') + + const files = await Promise.all( + Array.from({ length: 5 }, (_, i) => { + const p = path.join(scratchDir, `_measure_${i}.json`) + return fs.writeFile(p, '{}').then(() => p) + }) + ) + + // 1. chattr +i — single subprocess spawn, 5 sequential runs + const tChattr1: number[] = [] + for (const f of files) { + const t = process.hrtime.bigint() + await execa('chattr', ['+i', f]) + tChattr1.push(Number(process.hrtime.bigint() - t) / 1e6) + await execa('chattr', ['-i', f]) + } + console.log( + ` chattr +i single spawn: avg ${average(tChattr1).toFixed(1)} ms` + + ` [${tChattr1.map(t => t.toFixed(1)).join(', ')} ms]` + ) + + // 2. 5 concurrent chattr +i — realistic per-backup case (1 json + 4 vhds) + const tChattr5: number[] = [] + for (let run = 0; run < 5; run++) { + const t = process.hrtime.bigint() + await Promise.all(files.map(f => execa('chattr', ['+i', f]))) + tChattr5.push(Number(process.hrtime.bigint() - t) / 1e6) + await Promise.all(files.map(f => execa('chattr', ['-i', f]))) + } + console.log( + ` 5x concurrent chattr +i: avg ${average(tChattr5).toFixed(1)} ms` + + ` [${tChattr5.map(t => t.toFixed(1)).join(', ')} ms]` + ) + + await Promise.all(files.map(f => fs.unlink(f))) + console.log('') +} + +// --------------------------------------------------------------------------- +// Memory tracking +// --------------------------------------------------------------------------- + +let peakRss = 0 + +function sampleMemory(): number { + const { rss } = process.memoryUsage() + if (rss > peakRss) peakRss = rss + return rss +} + +function mb(bytes: number): string { + return (bytes / 1024 / 1024).toFixed(1) + ' MB' +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const root = await fs.mkdtemp(path.join(tmpdir(), 'immut-load-')) + + console.log(`Root: ${root}`) + console.log(`VMs: ${VM_COUNT}`) + console.log(`Disks per VM: ${DISKS_PER_VM}`) + console.log(`Files per VM: ${FILES_PER_VM}`) + console.log(`Expected total: ${EXPECTED_TOTAL}`) + console.log('') + + const vms = Array.from({ length: VM_COUNT }, (_, i) => ({ + uuid: makeUuid(i), + vdiUuids: Array.from({ length: DISKS_PER_VM }, (_, j) => makeUuid(VM_COUNT + i * DISKS_PER_VM + j)), + })) + const vmUuids = vms.map(vm => vm.uuid) + + // ------------------------------------------------------------------------- + // 1. Create directory structure + // ------------------------------------------------------------------------- + console.log('Creating directory structure...') + const tDirs = Date.now() + + await asyncEach( + vms, + async vm => { + await Promise.all( + vm.vdiUuids.map(vdiUuid => + fs.mkdir(path.join(root, 'xo-vm-backups', vm.uuid, 'vdis', JOB_UUID, vdiUuid), { + recursive: true, + }) + ) + ) + }, + { concurrency: 200 } + ) + console.log(`Done in ${Date.now() - tDirs} ms (${VM_COUNT * DISKS_PER_VM} leaf dirs)`) + console.log('') + + // ------------------------------------------------------------------------- + // 1.5. Baseline: measure raw operation costs (no watcher, no contention) + // ------------------------------------------------------------------------- + await measureRawCosts(path.join(root, 'xo-vm-backups', vms[0].uuid)) + + // ------------------------------------------------------------------------- + // 2. Start memory sampling + watchRemote + // ------------------------------------------------------------------------- + const memSampler = setInterval(sampleMemory, 50) + sampleMemory() + const rssBeforeWatch = process.memoryUsage().rss + console.log(`Memory before watchRemote: ${mb(rssBeforeWatch)}`) + + const tWatchStart = Date.now() + const { close } = await watchRemote('load-test', { + root, + immutabilityDuration: ONE_DAY_MS, + delayBetweenSizeCheck: 100, + }) + const tWatcherReady = Date.now() - tWatchStart + + await new Promise(resolve => setTimeout(resolve, 500)) + sampleMemory() + const rssAfterWatch = process.memoryUsage().rss + console.log(`watchRemote ready in ${tWatcherReady} ms`) + console.log(`Memory after watchRemote: ${mb(rssAfterWatch)} (delta: ${mb(rssAfterWatch - rssBeforeWatch)})`) + console.log('') + + // ------------------------------------------------------------------------- + // Phase A — single-backup probe: watcher running, zero concurrent load. + // Measures json-written → immutable latency for 5 sequential backups on a + // dedicated probe VM (not in the main 5000). + // Together with the baseline this isolates: event delivery + readdir overhead + // = (Phase A latency) − (5× concurrent chattr baseline). + // ------------------------------------------------------------------------- + console.log('=== Phase A: single-backup probe latency (watcher running, no load) ===') + + // Use UUIDs that cannot collide with the main vms[] (all-f prefix). + const PROBE_VM_UUID = 'ffffffff-ffff-ffff-ffff-000000000001' + const probeVm = { + uuid: PROBE_VM_UUID, + vdiUuids: Array.from( + { length: DISKS_PER_VM }, + (_, j) => `ffffffff-ffff-ffff-ffff-${String(j + 2).padStart(12, '0')}` + ), + } + const probeVmDir = path.join(root, 'xo-vm-backups', PROBE_VM_UUID) + + for (const vdiUuid of probeVm.vdiUuids) { + await fs.mkdir(path.join(probeVmDir, 'vdis', JOB_UUID, vdiUuid), { recursive: true }) + } + // Give the watcher time to detect and start watching the new VM dir. + await new Promise(resolve => setTimeout(resolve, 500)) + + const probeSingleTimes: number[] = [] + for (let run = 0; run < 5; run++) { + const pDatetime = `2024011${run + 1}T120000Z` + + await Promise.all( + probeVm.vdiUuids.map(vdiUuid => + fs.writeFile(path.join(probeVmDir, 'vdis', JOB_UUID, vdiUuid, `${pDatetime}.vhd`), 'fake vhd data') + ) + ) + + const jsonPath = path.join(probeVmDir, `${pDatetime}.json`) + const tJsonWritten = Date.now() + await fs.writeFile(jsonPath, '{}') + const tLocked = await waitUntilImmutable(jsonPath) + + if (tLocked !== null) { + const latency = tLocked - tJsonWritten + probeSingleTimes.push(latency) + console.log(` run ${run + 1}: json→immutable ${latency} ms`) + } else { + console.log(` run ${run + 1}: TIMEOUT`) + } + } + + if (probeSingleTimes.length > 0) { + const sorted = [...probeSingleTimes].sort((a, b) => a - b) + const avg = average(probeSingleTimes) + const med = sorted[Math.floor(sorted.length / 2)] + const max = sorted[sorted.length - 1] + console.log(` avg ${avg.toFixed(0)} ms median ${med} ms max ${max} ms`) + console.log(` (subtract 5x concurrent chattr baseline above to isolate fs.watch event latency + readdir)`) + } + console.log('') + + // ------------------------------------------------------------------------- + // Phase B — write all VM backups and measure live watcher locking speed. + // 5 probe VMs are tracked individually for per-backup latency under load. + // ------------------------------------------------------------------------- + console.log(`=== Phase B: live watcher — writing ${VM_COUNT} backups (${BACKUP_DATE}) ===`) + + const probeWriteNotifiers = new Map void>() + const probeBTasks = PROBE_VM_INDICES.map(vmIdx => { + const vm = vms[vmIdx] + const jsonPath = path.join(root, 'xo-vm-backups', vm.uuid, `${BACKUP_DATE}.json`) + return new Promise<{ vmIdx: number; tWritten: number; tLocked: number | null }>(resolve => { + probeWriteNotifiers.set(vm.uuid, tWritten => { + waitUntilImmutable(jsonPath).then(tLocked => resolve({ vmIdx, tWritten, tLocked })) + }) + }) + }) + + const tWrite = Date.now() + let written = 0 + + await asyncEach( + vms, + async vm => { + const vmDir = path.join(root, 'xo-vm-backups', vm.uuid) + await Promise.all( + vm.vdiUuids.map(vdiUuid => + fs.writeFile(path.join(vmDir, 'vdis', JOB_UUID, vdiUuid, `${BACKUP_DATE}.vhd`), 'fake vhd data') + ) + ) + await fs.writeFile(path.join(vmDir, `${BACKUP_DATE}.json`), '{}') + + written++ + probeWriteNotifiers.get(vm.uuid)?.(Date.now()) + + if (written % 1000 === 0) { + sampleMemory() + console.log( + ` ${written}/${VM_COUNT} backups written` + + ` | ${Date.now() - tWrite} ms elapsed` + + ` | peak RSS ${mb(peakRss)}` + ) + } + }, + { concurrency: WRITE_CONCURRENCY } + ) + const tWriteDone = Date.now() - tWrite + console.log(`All ${VM_COUNT} backups written in ${tWriteDone} ms`) + + console.log('\n Probe VM json→immutable latency (under full load):') + const [probeBResults, phaseB] = await Promise.all([ + Promise.all(probeBTasks), + waitForImmutableCount(root, BACKUP_DATE, vmUuids, VM_COUNT, 'live-lock'), + ]) + for (const { vmIdx, tWritten, tLocked } of probeBResults) { + const latency = tLocked === null ? 'TIMEOUT' : `${tLocked - tWritten} ms` + console.log(` VM[${String(vmIdx).padStart(4)}]: json→immutable ${latency}`) + } + + console.log(`Phase B ${phaseB.reached ? 'DONE' : 'TIMEOUT'} in ${phaseB.elapsed} ms`) + console.log('') + + // ------------------------------------------------------------------------- + // Phase C — lift all immutability + // ------------------------------------------------------------------------- + await close() + + console.log(`=== Phase C: lift all ${EXPECTED_TOTAL} protected files ===`) + const tLift = Date.now() + let lastLiftLog = Date.now() + + const liftSampler = setInterval(() => { + sampleMemory() + const now = Date.now() + if (now - lastLiftLog >= 5000) { + console.log(` lifting in progress... ${Date.now() - tLift} ms elapsed | peak RSS ${mb(peakRss)}`) + lastLiftLog = now + } + }, 1000) + + await liftRemoteImmutability(root, 0) + + clearInterval(liftSampler) + sampleMemory() + const tLiftDone = Date.now() - tLift + console.log(`Phase C DONE in ${tLiftDone} ms`) + console.log('') + + // ------------------------------------------------------------------------- + // Results + // ------------------------------------------------------------------------- + clearInterval(memSampler) + sampleMemory() + + const avgProbeSingle = probeSingleTimes.length > 0 ? `${average(probeSingleTimes).toFixed(0)} ms` : 'N/A' + const avgProbeBLatencies = probeBResults + .map(r => (r.tLocked === null ? null : r.tLocked - r.tWritten)) + .filter((v): v is number => v !== null) + const avgProbeB = avgProbeBLatencies.length > 0 ? `${average(avgProbeBLatencies).toFixed(0)} ms` : 'N/A' + + console.log('=== Results ===') + console.log(`watchRemote startup: ${tWatcherReady} ms`) + console.log(`Phase A — single probe avg: ${avgProbeSingle} (no contention)`) + console.log(`Phase B — write backups: ${tWriteDone} ms`) + console.log(`Phase B — lock all: ${phaseB.elapsed} ms ${phaseB.reached ? 'OK' : 'TIMEOUT'}`) + console.log(`Phase B — avg lock/backup: ${(phaseB.elapsed / VM_COUNT).toFixed(0)} ms (total / VM_COUNT)`) + console.log(`Phase B — probe avg (5 samples): ${avgProbeB} (under full load)`) + console.log(`Phase C — lift all: ${tLiftDone} ms`) + console.log(`Memory before watch: ${mb(rssBeforeWatch)}`) + console.log(`Memory after watch: ${mb(rssAfterWatch)}`) + console.log(`Peak RSS: ${mb(peakRss)}`) + console.log(`Current RSS: ${mb(process.memoryUsage().rss)}`) + + await cleanupRoot(root) +} + +main().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/@xen-orchestra/immutable-backups/src/remote.mts b/@xen-orchestra/immutable-backups/src/remote.mts new file mode 100644 index 0000000000..91e9684612 --- /dev/null +++ b/@xen-orchestra/immutable-backups/src/remote.mts @@ -0,0 +1,75 @@ +import fs from 'node:fs/promises' +import * as File from './file.mjs' +import assert from 'node:assert' +import { join } from 'node:path' +import { createLogger } from '@xen-orchestra/log' +import { type RemoteConfig } from './_loadConfig.mjs' +import { watchRemote as startRemoteWatcher } from './_watcher.mjs' + +const { debug, info, warn } = createLogger('xen-orchestra:immutable-backups:remote') + +// Verify that the remote filesystem supports immutability by creating, +// modifying, locking, and unlocking a temporary test file. +async function test(remotePath: string): Promise { + await fs.readdir(remotePath) + + const testPath = join(remotePath, '.test-immut') + // cleanup + try { + await File.liftImmutability(testPath) + await fs.unlink(testPath) + } catch { + // cleanup can fail if it's the first test — not an issue + } + // can create, modify and delete a file + await fs.writeFile(testPath, `test immut ${new Date()}`) + await fs.writeFile(testPath, `test immut change 1 ${new Date()}`) + await fs.unlink(testPath) + + // cannot modify or delete an immutable file + await fs.writeFile(testPath, `test immut ${new Date()}`) + await File.makeImmutable(testPath) + await assert.rejects(fs.writeFile(testPath, `test immut change 2 ${new Date()}`), { code: 'EPERM' }) + await assert.rejects(fs.unlink(testPath), { code: 'EPERM' }) + // can modify and delete a file after lifting immutability + await File.liftImmutability(testPath) + + await fs.writeFile(testPath, `test immut change 3 ${new Date()}`) + await fs.unlink(testPath) +} + +// Start watching a backup remote for new files and make them immutable as they are written. +export async function watchRemote( + remoteId: string, + { root, immutabilityDuration, delayBetweenSizeCheck }: RemoteConfig +): Promise<{ close: () => void }> { + debug('got config ', { remoteId, root, immutabilityDuration }) + + // test if fs supports immutability + await test(root) + + // Write the immutability settings file and lock it so it cannot be tampered with. + const settingPath = join(root, 'immutability.json') + // Lift first in case it was left immutable from a previous run. + try { + await fs.access(settingPath) + await File.liftImmutability(settingPath) + } catch (error) { + if (error.code !== 'ENOENT') { + info('error lifting immutability on current settings', { error }) + } + } + await fs.writeFile( + settingPath, + JSON.stringify({ + since: Date.now(), + immutable: true, + duration: immutabilityDuration, + }) + ) + await File.makeImmutable(settingPath) + + const close = await startRemoteWatcher(root, err => warn('watcher error', { err }), { delayBetweenSizeCheck }) + + return { close } +} diff --git a/@xen-orchestra/immutable-backups/tsconfig.json b/@xen-orchestra/immutable-backups/tsconfig.json new file mode 100644 index 0000000000..5e5dffa0d4 --- /dev/null +++ b/@xen-orchestra/immutable-backups/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + + "module": "ESNext", + "moduleResolution": "bundler", + "target": "es2022", + "skipLibCheck": true, + "types": ["node"], + "declaration": true + }, + "include": ["./src/**/*"], + "exclude": ["./node_modules", "./dist"] +} diff --git a/@xen-orchestra/log/index.d.ts b/@xen-orchestra/log/index.d.ts new file mode 100644 index 0000000000..0983ae0dfc --- /dev/null +++ b/@xen-orchestra/log/index.d.ts @@ -0,0 +1,24 @@ +/** + * A structured log entry's data payload. + * Typically contains contextual key-value pairs (e.g. `{ vmId, path }`). + */ +type LogData = Record + +/** + * A log method accepts either: + * - a message string with optional structured data + * - an Error instance (message is extracted, error is stored in data) + */ +type LogMethod = (message: string | Error | null, data?: LogData | Error | number | string | unknown) => void + +export interface Logger { + fatal: LogMethod + error: LogMethod + warn: LogMethod + info: LogMethod + debug: LogMethod + wrap: any>(message: string, fn: T) => T +} + +export function createLogger(namespace: string): Logger +export default createLogger diff --git a/@xen-orchestra/log/package.json b/@xen-orchestra/log/package.json index 67dbffbdfc..d3cb467477 100644 --- a/@xen-orchestra/log/package.json +++ b/@xen-orchestra/log/package.json @@ -36,6 +36,7 @@ "postversion": "npm publish", "test": "node --test" }, + "types": "./index.d.ts", "exports": { ".": "./index.js", "./_*": null, diff --git a/@xen-orchestra/rest-api/src/middlewares/generic-error-handler.middleware.mts b/@xen-orchestra/rest-api/src/middlewares/generic-error-handler.middleware.mts index 5928277836..b9c2237ae4 100644 --- a/@xen-orchestra/rest-api/src/middlewares/generic-error-handler.middleware.mts +++ b/@xen-orchestra/rest-api/src/middlewares/generic-error-handler.middleware.mts @@ -22,7 +22,7 @@ const log = createLogger('xo:rest-api:error-handler') // eslint-disable-next-line @typescript-eslint/no-unused-vars export default function genericErrorHandler(error: unknown, req: Request, res: Response, _next: NextFunction) { if (!(error instanceof Error)) { - log.error(error) + log.error(JSON.stringify(error)) res.status(500).json({ error }) return } diff --git a/@xen-orchestra/rest-api/src/vms/vm.service.mts b/@xen-orchestra/rest-api/src/vms/vm.service.mts index 7123198b9e..e5fd77b06c 100644 --- a/@xen-orchestra/rest-api/src/vms/vm.service.mts +++ b/@xen-orchestra/rest-api/src/vms/vm.service.mts @@ -130,7 +130,7 @@ export class VmService { nSuspended++ break default: - log.warn('Invalid VM power_state', vm.id, vm.power_state) + log.warn('Invalid VM power_state', {vmId: vm.id, vmPowerState : vm.power_state}) nUnknown++ break } diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 2b8d77a3e4..0eb02c748e 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -60,6 +60,7 @@ - @xen-orchestra/disk-cli major - @xen-orchestra/disk-transform patch - @xen-orchestra/fs minor +- @xen-orchestra/immutable-backups major - @xen-orchestra/mcp minor - @xen-orchestra/qcow2 minor - @xen-orchestra/rest-api minor diff --git a/docs/docs/immutability.md b/docs/docs/immutability.md index d651425909..c6d98fe6cf 100644 --- a/docs/docs/immutability.md +++ b/docs/docs/immutability.md @@ -50,6 +50,8 @@ npm install -g @xen-orchestra/immutable-backups Create a configuration file at `/etc/xo-immutable-backups/config.toml`, with the following structure: ```toml + +liftEvery = "1h" [remotes.remote1] root = "/mnt/ssd/vhdblock/" immutabilityDuration = "7d" @@ -57,18 +59,10 @@ immutabilityDuration = "7d" #### Mandatory Parameters -- **`root`**: Specifies the directory where Xen Orchestra stores backups. -- **`immutabilityDuration`**: Defines how long files remain protected from deletion (e.g., `7d` for 7 days). - -#### Optional Parameters - -For additional validation, you can enable: - -```toml -rebuildIndexOnStart = true -``` - -This option scans and validates existing files when the service starts. It can be resource-intensive. +- **`liftEvery`**: Define how often the script will check and lift immutabiltiy (e.g., `1h` for every hour). +- Per remote + - **`root`**: Specifies the directory where Xen Orchestra stores backups. + - **`immutabilityDuration`**: Defines how long files remain protected from deletion (e.g., `7d` for 7 days). ### 3. Start the Service diff --git a/packages/vhd-lib/disk-consumer/DiskConsumerVhdDirectory.mjs b/packages/vhd-lib/disk-consumer/DiskConsumerVhdDirectory.mjs index a0c5e29953..79643b2f61 100644 --- a/packages/vhd-lib/disk-consumer/DiskConsumerVhdDirectory.mjs +++ b/packages/vhd-lib/disk-consumer/DiskConsumerVhdDirectory.mjs @@ -6,12 +6,12 @@ */ import { BaseVhd, FULL_BLOCK_BITMAP } from './BaseVhd.mjs' -import { dirname } from 'node:path' -import { v4 as uuidv4 } from 'uuid' +import { basename, dirname } from 'node:path' import { asyncEach } from '@vates/async-each' import { VhdDirectory, VhdAbstract } from 'vhd-lib' import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' import { DEFAULT_BLOCK_SIZE } from '../_constants.js' +import assert from 'node:assert' /** * @typedef {Object} VhdRemoteTarget @@ -45,7 +45,10 @@ export class DiskConsumerVhdDirectory extends BaseVhd { */ async write(signal) { const { handler, path, compression, flags, validator, concurrency } = this.#target - const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd` + const SUFFIX = '.alias.vhd' + assert.ok(path.endsWith(SUFFIX), `filename must be an alias , got ${path}`) + const base = basename(path).substring(0, -SUFFIX.length) + const dataPath = `${dirname(path)}/data/${base}.vhd` const uid = 'to stream ' + Math.random() let generator try {