Administrator/Xen-Orchestra-xen-orchestra-MIRROR
1
0
Fork 1
mirror of https://github.com/vatesfr/xen-orchestra.git synced 2026-09-10 22:14:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(immutable-backups): decrease resource consumption, add tests and typing (#9548)

Browse Source
This commit is contained in:
Florent BEAUCHAMP
2026-03-24 16:00:01 +01:00
committed by GitHub
parent f2b1931408
commit 0286f7786a
38 changed files with 2041 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
@vates/async-each/index.test.js
View File

@@ -67,20 +67,17 @@ describe('asyncEach', () => {
})
it('stops on first error when stopOnError is true', async () => {
const tracker = new assert.CallTracker()
const error = new Error()
const iteratee = tracker.calls((_, i) => {
const iteratee = spy((_, i) => {
if (i === 1) {
throw error
}
}, 2)
})
assert.deepStrictEqual(
await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true })),
error
)
tracker.verify()
assert.deepStrictEqual(iteratee.callCount, 2)
})
it('rejects AggregateError when stopOnError is false', async () => {
@@ -100,19 +97,16 @@ describe('asyncEach', () => {
})
it('can be interrupted with an AbortSignal', async () => {
const tracker = new assert.CallTracker()
const ac = new AbortController()
const iteratee = tracker.calls((_, i) => {
const iteratee = spy((_, i) => {
if (i === 1) {
ac.abort()
}
}, 2)
})
await assert.rejects(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal }), {
message: 'asyncEach aborted',
})
tracker.verify()
assert.deepStrictEqual(iteratee.callCount, 2)
})
it('handle error on iterable', async () => {

2
@xen-orchestra/disk-transform/src/utils/DebugConsumer.mts
View File

@@ -13,7 +13,7 @@ export async function consume(disk: Disk, { blockDelay = 20, concurrency = 8 } =
await asyncEach(
disk.diskBlocks(),
async (block: DiskBlock) => {
async block => {
consumed += block.data.length
nbBlocks++
await new Promise(resolve => setTimeout(resolve, blockDelay))

10
@xen-orchestra/immutable-backups/_cleanXoCache.mjs
View File

@@ -1,10 +0,0 @@
import fs from 'node:fs/promises'
import { dirname, join } from 'node:path'
import isBackupMetadata from './isBackupMetadata.mjs'
export default async path => {
if (isBackupMetadata(path)) {
// snipe vm metadata cache to force XO to update it
await fs.unlink(join(dirname(path), 'cache.json.gz'))
}
}

4
@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs
View File

@@ -1,4 +0,0 @@
import { dirname } from 'node:path'
// check if we are handling file directly under a vhd directory ( bat, headr, footer,..)
export default path => dirname(path).endsWith('.vhd')

46
@xen-orchestra/immutable-backups/_loadConfig.mjs
View File

@@ -1,46 +0,0 @@
import { load } from 'app-conf'
import { homedir } from 'os'
import { join } from 'node:path'
import ms from 'ms'
const APP_NAME = 'xo-immutable-backups'
const APP_DIR = new URL('.', import.meta.url).pathname
export default async function loadConfig() {
const config = await load(APP_NAME, {
appDir: APP_DIR,
ignoreUnknownFormats: true,
})
if (config.remotes === undefined || config.remotes?.length < 1) {
throw new Error(
'No remotes are configured in the config file, please add at least one [remotes.<remoteid>] with a root property pointing to the absolute path of the remote to watch'
)
}
if (config.liftEvery) {
config.liftEvery = ms(config.liftEvery)
}
for (const [remoteId, { indexPath, immutabilityDuration, root }] of Object.entries(config.remotes)) {
if (!root) {
throw new Error(
`Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository `
)
}
if (!immutabilityDuration) {
throw new Error(
`Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be protected by immutability `
)
}
if (ms(immutabilityDuration) < ms('1d')) {
throw new Error(
`Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}`
)
}
if (!indexPath) {
const basePath = indexPath ?? process.env.XDG_DATA_HOME ?? join(homedir(), '.local', 'share')
const immutabilityIndexPath = join(basePath, APP_NAME, remoteId)
config.remotes[remoteId].indexPath = immutabilityIndexPath
}
config.remotes[remoteId].immutabilityDuration = ms(immutabilityDuration)
}
return config
}

33
@xen-orchestra/immutable-backups/directory.integ.mjs
View File

@@ -1,33 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import { tmpdir } from 'node:os'
import * as Directory from './directory.mjs'
import { rimraf } from 'rimraf'
describe('immutable-backups/file', async () => {
it('really lock a directory', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
const dataDir = path.join(dir, 'data')
await fs.mkdir(dataDir)
const immutDir = path.join(dir, '.immutable')
const filePath = path.join(dataDir, 'test')
await fs.writeFile(filePath, 'data')
await Directory.makeImmutable(dataDir, immutDir)
assert.strictEqual(await Directory.isImmutable(dataDir), true)
await assert.rejects(() => fs.writeFile(filePath, 'data'))
await assert.rejects(() => fs.appendFile(filePath, 'data'))
await assert.rejects(() => fs.unlink(filePath))
await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data'))
await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy'))
await Directory.liftImmutability(dataDir, immutDir)
assert.strictEqual(await Directory.isImmutable(dataDir), false)
await fs.writeFile(filePath, 'data')
await fs.appendFile(filePath, 'data')
await fs.unlink(filePath)
await fs.rename(dataDir, dataDir + 'copy')
await rimraf(dir)
})
})

21
@xen-orchestra/immutable-backups/directory.mjs
View File

@@ -1,21 +0,0 @@
import execa from 'execa'
import { unindexFile, indexFile } from './fileIndex.mjs'
export async function makeImmutable(dirPath, immutabilityCachePath) {
if (immutabilityCachePath) {
await indexFile(dirPath, immutabilityCachePath)
}
await execa('chattr', ['+i', '-R', dirPath])
}
export async function liftImmutability(dirPath, immutabilityCachePath) {
if (immutabilityCachePath) {
await unindexFile(dirPath, immutabilityCachePath)
}
await execa('chattr', ['-i', '-R', dirPath])
}
export async function isImmutable(path) {
const { stdout } = await execa('lsattr', ['-d', path])
return stdout[4] === 'i'
}

24
@xen-orchestra/immutable-backups/file.mjs
View File

@@ -1,24 +0,0 @@
import execa from 'execa'
import { unindexFile, indexFile } from './fileIndex.mjs'
// this work only on linux like systems
// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/
export async function makeImmutable(path, immutabilityCachePath) {
if (immutabilityCachePath) {
await indexFile(path, immutabilityCachePath)
}
await execa('chattr', ['+i', path])
}
export async function liftImmutability(filePath, immutabilityCachePath) {
if (immutabilityCachePath) {
await unindexFile(filePath, immutabilityCachePath)
}
await execa('chattr', ['-i', filePath])
}
export async function isImmutable(path) {
const { stdout } = await execa('lsattr', ['-d', path])
return stdout[4] === 'i'
}

81
@xen-orchestra/immutable-backups/fileIndex.integ.mjs
View File

@@ -1,81 +0,0 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import * as FileIndex from './fileIndex.mjs'
import * as Directory from './directory.mjs'
import { tmpdir } from 'node:os'
import { rimraf } from 'rimraf'
describe('immutable-backups/fileIndex', async () => {
it('index File changes', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
const immutDir = path.join(dir, '.immutable')
const filePath = path.join(dir, 'test.ext')
await fs.writeFile(filePath, 'data')
await FileIndex.indexFile(filePath, immutDir)
await fs.mkdir(path.join(immutDir, 'NOTADATE'))
await fs.writeFile(path.join(immutDir, 'NOTADATE.file'), 'content')
let nb = 0
let index, target
for await ({ index, target } of FileIndex.listOlderTargets(immutDir, 0)) {
assert.strictEqual(true, false, 'Nothing should be eligible for deletion')
}
nb = 0
for await ({ index, target } of FileIndex.listOlderTargets(immutDir, -24 * 60 * 60 * 1000)) {
assert.strictEqual(target, filePath)
await fs.unlink(index)
nb++
}
assert.strictEqual(nb, 1)
await fs.rmdir(path.join(immutDir, 'NOTADATE'))
await fs.rm(path.join(immutDir, 'NOTADATE.file'))
for await ({ index, target } of FileIndex.listOlderTargets(immutDir, -24 * 60 * 60 * 1000)) {
// should remove the empty dir
assert.strictEqual(true, false, 'Nothing should have stayed here')
}
assert.strictEqual((await fs.readdir(immutDir)).length, 0)
await rimraf(dir)
})
it('fails correctly', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
const immutDir = path.join(dir, '.immutable')
await fs.mkdir(immutDir)
const placeholderFile = path.join(dir, 'test.ext')
await fs.writeFile(placeholderFile, 'data')
await FileIndex.indexFile(placeholderFile, immutDir)
const filePath = path.join(dir, 'test2.ext')
await fs.writeFile(filePath, 'data')
await FileIndex.indexFile(filePath, immutDir)
await assert.rejects(() => FileIndex.indexFile(filePath, immutDir), { code: 'EEXIST' })
await Directory.makeImmutable(immutDir)
await assert.rejects(() => FileIndex.unindexFile(filePath, immutDir), { code: 'EPERM' })
await Directory.liftImmutability(immutDir)
await rimraf(dir)
})
it('handles bomb index files', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
const immutDir = path.join(dir, '.immutable')
await fs.mkdir(immutDir)
const placeholderFile = path.join(dir, 'test.ext')
await fs.writeFile(placeholderFile, 'data')
await FileIndex.indexFile(placeholderFile, immutDir)
const indexDayDir = path.join(immutDir, '1980,11-28')
await fs.mkdir(indexDayDir)
await fs.writeFile(path.join(indexDayDir, 'big'), Buffer.alloc(2 * 1024 * 1024))
assert.rejects(async () => {
let index, target
for await ({ index, target } of FileIndex.listOlderTargets(immutDir, 0)) {
// should remove the empty dir
assert.strictEqual(true, false, `Nothing should have stayed here, got ${index} ${target}`)
}
})
await rimraf(dir)
})
})

88
@xen-orchestra/immutable-backups/fileIndex.mjs
View File

@@ -1,88 +0,0 @@
import { join } from 'node:path'
import { createHash } from 'node:crypto'
import fs from 'node:fs/promises'
import { dirname } from 'path'
const MAX_INDEX_FILE_SIZE = 1024 * 1024
function sha256(content) {
return createHash('sha256').update(content).digest('hex')
}
function formatDate(date) {
return date.toISOString().split('T')[0]
}
async function computeIndexFilePath(path, immutabilityIndexPath) {
const stat = await fs.stat(path)
const date = new Date(stat.birthtimeMs)
const day = formatDate(date)
const hash = sha256(path)
return join(immutabilityIndexPath, day, hash)
}
export async function indexFile(path, immutabilityIndexPath) {
const indexFilePath = await computeIndexFilePath(path, immutabilityIndexPath)
try {
await fs.writeFile(indexFilePath, path, { flag: 'wx' })
} catch (err) {
// missing dir: make it
if (err.code === 'ENOENT') {
await fs.mkdir(dirname(indexFilePath), { recursive: true })
await fs.writeFile(indexFilePath, path)
} else {
throw err
}
}
return indexFilePath
}
export async function unindexFile(path, immutabilityIndexPath) {
try {
const cacheFileName = await computeIndexFilePath(path, immutabilityIndexPath)
await fs.unlink(cacheFileName)
} catch (err) {
if (err.code !== 'ENOENT') {
throw err
}
}
}
export async function* listOlderTargets(immutabilityCachePath, immutabilityDuration) {
// walk all dir by day until the limit day
const limitDate = new Date(Date.now() - immutabilityDuration)
const limitDay = formatDate(limitDate)
const dir = await fs.opendir(immutabilityCachePath)
for await (const dirent of dir) {
if (dirent.isFile()) {
continue
}
// ensure we have a valid date
if (isNaN(new Date(dirent.name))) {
continue
}
// recent enough to be kept
if (dirent.name >= limitDay) {
continue
}
const subDirPath = join(immutabilityCachePath, dirent.name)
const subdir = await fs.opendir(subDirPath)
let nb = 0
for await (const hashFileEntry of subdir) {
const entryFullPath = join(subDirPath, hashFileEntry.name)
const { size } = await fs.stat(entryFullPath)
if (size > MAX_INDEX_FILE_SIZE) {
throw new Error(`Index file at ${entryFullPath} is too big, ${size} bytes `)
}
const targetPath = await fs.readFile(entryFullPath, { encoding: 'utf8' })
yield {
index: entryFullPath,
target: targetPath,
}
nb++
}
// cleanup older folder
if (nb === 0) {
await fs.rmdir(subDirPath)
}
}
}

1
@xen-orchestra/immutable-backups/isBackupMetadata.mjs
View File

@@ -1 +0,0 @@
export default path => path.match(/xo-vm-backups\/[^/]+\/[^/]+\.json$/)

35
@xen-orchestra/immutable-backups/liftProtection.mjs
View File

@@ -1,35 +0,0 @@
#!/usr/bin/env node
import * as Directory from './directory.mjs'
import { createLogger } from '@xen-orchestra/log'
import { listOlderTargets } from './fileIndex.mjs'
import cleanXoCache from './_cleanXoCache.mjs'
import loadConfig from './_loadConfig.mjs'
const { info, warn } = createLogger('xen-orchestra:immutable-backups:liftProtection')
async function liftRemoteImmutability(immutabilityCachePath, immutabilityDuration) {
for await (const { target } of listOlderTargets(immutabilityCachePath, immutabilityDuration)) {
await Directory.liftImmutability(target, immutabilityCachePath)
await cleanXoCache(target)
}
}
async function liftImmutability(remotes) {
for (const [remoteId, { indexPath, immutabilityDuration }] of Object.entries(remotes)) {
liftRemoteImmutability(indexPath, immutabilityDuration).catch(err =>
warn('error during watchRemote', { err, remoteId, indexPath, immutabilityDuration })
)
}
}
const { liftEvery, remotes } = await loadConfig()
if (liftEvery > 0) {
info('setup watcher for immutability lifting')
setInterval(async () => {
liftImmutability(remotes)
}, liftEvery)
} else {
liftImmutability(remotes)
}

17
@xen-orchestra/immutable-backups/package.json
View File

@@ -13,30 +13,31 @@
"url": "https://vates.fr"
},
"bin": {
"xo-immutable-remote": "./protectRemotes.mjs",
"xo-lift-remote-immutability": "./liftProtection.mjs"
"xo-immutable-remote": "./dist/protectRemotes.mjs"
},
"license": "AGPL-3.0-or-later",
"version": "1.0.30",
"engines": {
"node": ">=14.0.0"
"node": ">=20.0.0"
},
"dependencies": {
"@vates/async-each": "^1.0.1",
"@xen-orchestra/backups": "^0.69.4",
"@xen-orchestra/log": "^0.7.1",
"app-conf": "^3.0.0",
"chokidar": "^3.6.0",
"execa": "^5.0.0",
"ms": "^2.1.3",
"vhd-lib": "^4.14.7"
"ms": "^2.1.3"
},
"devDependencies": {
"@types/node": "^22.3.0",
"rimraf": "^6.0.1",
"tap": "^18.6.1"
"tap": "^18.6.1",
"typescript": "~5.6"
},
"scripts": {
"build": "tsc",
"clean": "rimraf dist/",
"postversion": "npm publish --access public",
"test-integration": "tap *.integ.mjs"
"test-integration": "tsc && node --test dist/*.integ.mjs"
}
}

193
@xen-orchestra/immutable-backups/protectRemotes.mjs
View File

@@ -1,193 +0,0 @@
#!/usr/bin/env node
import fs from 'node:fs/promises'
import * as File from './file.mjs'
import * as Directory from './directory.mjs'
import assert from 'node:assert'
import { dirname, join, sep } from 'node:path'
import { createLogger } from '@xen-orchestra/log'
import chokidar from 'chokidar'
import { indexFile } from './fileIndex.mjs'
import cleanXoCache from './_cleanXoCache.mjs'
import loadConfig from './_loadConfig.mjs'
import isInVhdDirectory from './_isInVhdDirectory.mjs'
const { debug, info, warn } = createLogger('xen-orchestra:immutable-backups:remote')
async function test(remotePath, indexPath) {
await fs.readdir(remotePath)
const testPath = join(remotePath, '.test-immut')
// cleanup
try {
await File.liftImmutability(testPath, indexPath)
await fs.unlink(testPath)
} catch (err) {}
// can create , modify and delete a file
await fs.writeFile(testPath, `test immut ${new Date()}`)
await fs.writeFile(testPath, `test immut change 1 ${new Date()}`)
await fs.unlink(testPath)
// cannot modify or delete an immutable file
await fs.writeFile(testPath, `test immut ${new Date()}`)
await File.makeImmutable(testPath, indexPath)
await assert.rejects(fs.writeFile(testPath, `test immut change 2 ${new Date()}`), { code: 'EPERM' })
await assert.rejects(fs.unlink(testPath), { code: 'EPERM' })
// can modify and delete a file after lifting immutability
await File.liftImmutability(testPath, indexPath)
await fs.writeFile(testPath, `test immut change 3 ${new Date()}`)
await fs.unlink(testPath)
}
async function handleExistingFile(root, indexPath, path) {
try {
// a vhd block directory is completely immutable
if (isInVhdDirectory(path)) {
// this will trigger 3 times per vhd blocks
const dir = join(root, dirname(path))
if (Directory.isImmutable(dir)) {
await indexFile(dir, indexPath)
}
} else {
// other files are immutable a file basis
const fullPath = join(root, path)
if (File.isImmutable(fullPath)) {
await indexFile(fullPath, indexPath)
}
}
} catch (error) {
if (error.code !== 'EEXIST') {
// there can be a symbolic link in the tree
warn('handleExistingFile', { error })
}
}
}
async function handleNewFile(root, indexPath, pendingVhds, path) {
// with awaitWriteFinish we have complete files here
// we can make them immutable
if (isInVhdDirectory(path)) {
// watching a vhd block
// wait for header/footer and BAT before making this immutable recursively
const splitted = path.split(sep)
const vmUuid = splitted[1]
const vdiUuid = splitted[4]
const uniqPath = `${vmUuid}/${vdiUuid}`
const { existing } = pendingVhds.get(uniqPath) ?? {}
if (existing === undefined) {
pendingVhds.set(uniqPath, { existing: 1, lastModified: Date.now() })
} else {
// already two of the key files,and we got the last one
if (existing === 2) {
await Directory.makeImmutable(join(root, dirname(path)), indexPath)
pendingVhds.delete(uniqPath)
} else {
// wait for the other
pendingVhds.set(uniqPath, { existing: existing + 1, lastModified: Date.now() })
}
}
} else {
const fullFilePath = join(root, path)
await File.makeImmutable(fullFilePath, indexPath)
await cleanXoCache(fullFilePath)
}
}
export async function watchRemote(remoteId, { root, immutabilityDuration, rebuildIndexOnStart = false, indexPath }) {
// create index directory
await fs.mkdir(indexPath, { recursive: true })
// test if fs and index directories are well configured
await test(root, indexPath)
// add duration and watch status in the metadata.json of the remote
const settingPath = join(root, 'immutability.json')
try {
// this file won't be made mutable by liftimmutability
await File.liftImmutability(settingPath)
} catch (error) {
// file may not exists, and it's not really a problem
info('lifting immutability on current settings', { error })
}
await fs.writeFile(
settingPath,
JSON.stringify({
since: Date.now(),
immutable: true,
duration: immutabilityDuration,
})
)
// no index path in makeImmutable(): the immutability won't be lifted
File.makeImmutable(settingPath)
// we wait for footer/header AND BAT to be written before locking a vhd directory
// this map allow us to track the vhd with partial metadata
const pendingVhds = new Map()
// cleanup pending vhd map periodically
setInterval(
() => {
pendingVhds.forEach(({ lastModified, existing }, path) => {
if (Date.now() - lastModified > 60 * 60 * 1000) {
pendingVhds.delete(path)
warn(`vhd at ${path} is incomplete since ${lastModified}`, { existing, lastModified, path })
}
})
},
60 * 60 * 1000
)
// watch the remote for any new VM metadata json file
const PATHS = [
// xo-config-backups/scheduleId/date/metadata.json
'xo-config-backups/*/*/data',
'xo-config-backups/*/*/data.json',
'xo-config-backups/*/*/metadata.json',
// xo-pool-metadata-backups/backupId/scheduleId/date/metadata.json
'xo-pool-metadata-backups/*/*/*/metadata.json',
'xo-pool-metadata-backups/*/*/*/data',
// xo-vm-backups/<vmuuid>/
'xo-vm-backups/*/*.json',
'xo-vm-backups/*/*.xva',
'xo-vm-backups/*/*.xva.checksum',
// xo-vm-backups/<vmuuid>/vdis/<jobid>/<vdiUuid>
'xo-vm-backups/*/vdis/*/*/*.vhd', // can be an alias or a vhd file
// for vhd directory :
'xo-vm-backups/*/vdis/*/*/data/*.vhd/bat',
'xo-vm-backups/*/vdis/*/*/data/*.vhd/header',
'xo-vm-backups/*/vdis/*/*/data/*.vhd/footer',
]
let ready = false
const watcher = chokidar.watch(PATHS, {
ignored: [
/(^|[/\\])\../, // ignore dotfiles
/\.lock$/,
],
cwd: root,
recursive: false, // vhd directory can generate a lot of folder, don't let chokidar choke on this
ignoreInitial: !rebuildIndexOnStart,
depth: 7,
awaitWriteFinish: true,
})
// Add event listeners.
watcher
.on('add', async path => {
debug(`File ${path} has been added ${path.split('/').length}`)
if (ready) {
await handleNewFile(root, indexPath, pendingVhds, path)
} else {
await handleExistingFile(root, indexPath, path)
}
})
.on('error', error => warn(`Watcher error: ${error}`))
.on('ready', () => {
ready = true
info('Ready for changes')
})
}
const { remotes } = await loadConfig()
for (const [remoteId, remote] of Object.entries(remotes)) {
watchRemote(remoteId, remote).catch(err => warn('error during watchRemote', { err, remoteId, remote }))
}

16
@xen-orchestra/immutable-backups/src/_cleanXoCache.mts Normal file
View File

@@ -0,0 +1,16 @@
import fs from 'node:fs/promises'
import { dirname, join } from 'node:path'
import isBackupMetadata from './isBackupMetadata.mjs'
// If `path` is a VM backup metadata file, delete the adjacent `cache.json.gz`
// so that XO re-reads the updated metadata on next access.
export default async (path: string): Promise<void> => {
if (isBackupMetadata(path)) {
// snipe vm metadata cache to force XO to update it
await fs.unlink(join(dirname(path), 'cache.json.gz')).catch(err => {
if (err.code !== 'ENOENT') {
throw err
}
})
}
}

100
@xen-orchestra/immutable-backups/src/_loadConfig.integ.mts Normal file
View File

@@ -0,0 +1,100 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import type { StringValue } from 'ms'
import { parseConfig } from './_loadConfig.mjs'
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
const VALID_REMOTE = { root: '/fake/root', immutabilityDuration: '30d' as StringValue }
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('parseConfig', () => {
it('throws when remotes key is absent', () => {
assert.throws(() => parseConfig({ liftEvery: '1h' } as any), /No remotes are configured/)
})
it('throws when a remote is missing the root property', () => {
assert.throws(
() => parseConfig({ liftEvery: '1h', remotes: { myRemote: { immutabilityDuration: '30d' } } }),
/don't have a root property/
)
})
it('throws when a remote is missing immutabilityDuration', () => {
assert.throws(
() => parseConfig({ liftEvery: '1h', remotes: { myRemote: { root: '/fake/root' } } }),
/don't have a immutabilityDuration/
)
})
it('throws when immutabilityDuration is shorter than 1 day', () => {
assert.throws(
() =>
parseConfig({ liftEvery: '1h', remotes: { myRemote: { root: '/fake/root', immutabilityDuration: '23h' } } }),
/smaller than the minimum/
)
})
// --- liftEvery validation ---
it('throws when liftEvery is a number', () => {
assert.throws(
() => parseConfig({ liftEvery: 3600 as any, remotes: { myRemote: VALID_REMOTE } }),
/must be a string for liftEvery/
)
})
it('throws when liftEvery is an unparsable string', () => {
assert.throws(
() => parseConfig({ liftEvery: 'notaduration' as any, remotes: { myRemote: VALID_REMOTE } }),
/is not a valid value for entry liftEvery/
)
})
// --- immutabilityDuration validation ---
it('throws when immutabilityDuration is a number', () => {
assert.throws(
() =>
parseConfig({
liftEvery: '1h',
remotes: { myRemote: { root: '/fake/root', immutabilityDuration: 86400000 as any } },
}),
/must be a string for remote myRemote/
)
})
it('throws when immutabilityDuration is an unparsable string', () => {
assert.throws(
() =>
parseConfig({
liftEvery: '1h',
remotes: { myRemote: { root: '/fake/root', immutabilityDuration: 'notaduration' as any } },
}),
/is not a valid value for entry immutabilityDuration/
)
})
it('parses a valid duration string to milliseconds', () => {
const config = parseConfig({
liftEvery: '1h',
remotes: { myRemote: { root: '/some/path', immutabilityDuration: '30d' } },
})
assert.strictEqual(config.remotes['myRemote'].immutabilityDuration, 30 * 24 * 60 * 60 * 1000)
})
it('parses liftEvery duration string to milliseconds', () => {
const config = parseConfig({
liftEvery: '1h',
remotes: { myRemote: { root: '/some/path', immutabilityDuration: '7d' } },
})
assert.strictEqual(config.liftEvery, 60 * 60 * 1000)
})
})

98
@xen-orchestra/immutable-backups/src/_loadConfig.mts Normal file
View File

@@ -0,0 +1,98 @@
import { load } from 'app-conf'
import ms, { StringValue } from 'ms'
// Configuration for a single watched remote.
// All duration fields are stored as milliseconds after config loading.
export interface RemoteConfig {
/** Absolute path to the root of the backup repository */
root: string
/** Minimum duration in milliseconds that files must stay immutable */
immutabilityDuration: number
/** Milliseconds to wait between consecutive size checks when polling for write completion (default: 100) */
delayBetweenSizeCheck: number
}
export interface AppConfigInput {
liftEvery: StringValue
/** Map of remote ID to its resolved configuration */
remotes: Record<
string,
Partial<{ root: string; immutabilityDuration: StringValue; delayBetweenSizeCheck: StringValue }>
>
}
export interface AppConfig {
/** Interval in milliseconds between immutability-lifting runs */
liftEvery: number
/** Map of remote ID to its resolved configuration */
remotes: Record<string, RemoteConfig>
}
const APP_NAME = 'xo-immutable-backups'
const APP_DIR = new URL('.', import.meta.url).pathname
// Validate and transform a raw config object (as returned by app-conf) into a
// typed AppConfig. Duration strings are resolved to milliseconds.
// Exported so it can be unit-tested without touching the filesystem.
export function parseConfig(config: AppConfigInput): AppConfig {
const outputConfig: AppConfig = {
liftEvery: 0,
remotes: {},
}
if (config.remotes === undefined || Object.keys(config.remotes).length < 1) {
throw new Error(
'No remotes are configured in the config file, please add at least one [remotes.<remoteid>] with a root property pointing to the absolute path of the remote to watch'
)
}
if (typeof config.liftEvery !== 'string') {
throw new Error(`${config.liftEvery} must be a string for liftEvery`)
}
outputConfig.liftEvery = ms(config.liftEvery)
if (isNaN(outputConfig.liftEvery)) {
throw new Error(`${config.liftEvery} is not a valid value for entry liftEvery`)
}
for (const [remoteId, { immutabilityDuration, root, delayBetweenSizeCheck }] of Object.entries(config.remotes)) {
if (!root) {
throw new Error(
`Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository `
)
}
if (!immutabilityDuration) {
throw new Error(
`Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be protected by immutability `
)
}
if (typeof immutabilityDuration !== 'string') {
throw new Error(`${immutabilityDuration} is must be a string for remote ${remoteId}`)
}
if (isNaN(ms(immutabilityDuration))) {
throw new Error(
`${immutabilityDuration} is not a valid value for entry immutabilityDuration for remote ${remoteId}`
)
}
if (ms(immutabilityDuration) < ms('1d')) {
throw new Error(
`Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}`
)
}
outputConfig.remotes[remoteId] = {
immutabilityDuration: ms(immutabilityDuration),
root,
delayBetweenSizeCheck: delayBetweenSizeCheck !== undefined ? ms(delayBetweenSizeCheck) : 100,
}
}
return outputConfig
}
// Load the raw configuration from disk via app-conf.
export function loadRawConfig(): Promise<AppConfigInput> {
return load(APP_NAME, {
appDir: APP_DIR,
ignoreUnknownFormats: true,
})
}
// Convenience default export: load from disk and parse in one call.
export default async function loadConfig(): Promise<AppConfig> {
return parseConfig(await loadRawConfig())
}

219
@xen-orchestra/immutable-backups/src/_watcher.integ.mts Normal file
View File

@@ -0,0 +1,219 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import { tmpdir } from 'node:os'
import { rimraf } from 'rimraf'
import execa from 'execa'
import * as File from './file.mjs'
import { waitForWriteDone, watchVmDirectory } from './_watcher.mjs'
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
async function mkTmp(): Promise<string> {
return fs.mkdtemp(path.join(tmpdir(), 'watcher-test-'))
}
async function cleanupRoot(root: string): Promise<void> {
try {
await execa('chattr', ['-i', '-R', root])
} catch (_err) {}
await rimraf(root)
}
// ---------------------------------------------------------------------------
// waitForWriteDone
// ---------------------------------------------------------------------------
describe('waitForWriteDone', () => {
it('resolves immediately when file size is already stable', async () => {
const dir = await mkTmp()
try {
const file = path.join(dir, 'stable.bin')
await fs.writeFile(file, 'hello')
// Two polls must see the same size — give it 1 s timeout, should resolve fast
await waitForWriteDone(file, 1000, 100)
} finally {
await rimraf(dir)
}
})
it('waits for ENOENT then resolves once file appears with stable size', async () => {
const dir = await mkTmp()
try {
const file = path.join(dir, 'late.bin')
// Write the file 150 ms after the poll starts
setTimeout(async () => {
await fs.writeFile(file, 'data')
}, 150)
await waitForWriteDone(file, 2000, 100)
} finally {
await rimraf(dir)
}
})
it('does NOT resolve on an empty file (size === 0)', async () => {
const dir = await mkTmp()
try {
const file = path.join(dir, 'empty.bin')
await fs.writeFile(file, '')
// The function should time out because size is 0 — use a short timeout
await assert.rejects(waitForWriteDone(file, 400, 100), /Timeout/)
} finally {
await rimraf(dir)
}
})
it('rejects with Timeout when file never stabilises', async () => {
const dir = await mkTmp()
try {
const file = path.join(dir, 'growing.bin')
// Keep appending so size never stabilises
let stop = false
const writer = (async () => {
let i = 0
while (!stop) {
await fs.appendFile(file, `chunk-${i++}\n`)
await new Promise(r => setTimeout(r, 80))
}
})()
await assert.rejects(waitForWriteDone(file, 500, 100), /Timeout/)
stop = true
await writer
} finally {
await rimraf(dir)
}
})
it('resolves once a growing file stops changing', async () => {
const dir = await mkTmp()
try {
const file = path.join(dir, 'finish.bin')
await fs.writeFile(file, 'chunk1')
// After 200 ms stop appending — function should resolve before 2 s
setTimeout(async () => {
await fs.appendFile(file, 'chunk2')
}, 50)
// No more writes after that; the size stabilises
await waitForWriteDone(file, 2000, 100)
} finally {
await rimraf(dir)
}
})
})
// ---------------------------------------------------------------------------
// watchVmDirectory
// ---------------------------------------------------------------------------
const VM_UUID = 'aaaaaaaa-0000-0000-0000-000000000001'
const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001'
const VDI_UUID = 'cccccccc-0000-0000-0000-000000000001'
const BACKUP_DATE = '20240115T120000Z'
const ONE_DAY_MS = 24 * 60 * 60 * 1000
async function waitFor(fn: () => Promise<boolean>, timeout = 8000): Promise<void> {
const deadline = Date.now() + timeout
while (Date.now() < deadline) {
try {
if (await fn()) return
} catch {}
await new Promise(resolve => setTimeout(resolve, 200))
}
throw new Error('Timed out waiting for condition')
}
describe('watchVmDirectory', () => {
it('ignores non-.json files', async () => {
const dir = await mkTmp()
const vmDir = path.join(dir, 'vm')
await fs.mkdir(vmDir, { recursive: true })
const errors: unknown[] = []
const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 })
try {
// Write a file that doesn't match *.json
const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`)
await fs.writeFile(xvaFile, 'fake xva')
// Give the watcher time to react
await new Promise(r => setTimeout(r, 500))
// Must not be immutable — no .json was written
assert.strictEqual(await File.isImmutable(xvaFile), false, '.xva must stay mutable without .json')
assert.strictEqual(errors.length, 0, 'no errors expected')
} finally {
close()
await cleanupRoot(dir)
}
})
it('ignores json files that do not match datetime prefix (e.g. cache.json.gz)', async () => {
const dir = await mkTmp()
const vmDir = path.join(dir, 'vm')
await fs.mkdir(vmDir, { recursive: true })
const errors: unknown[] = []
const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 })
try {
const cacheFile = path.join(vmDir, 'cache.json')
await fs.writeFile(cacheFile, '{}')
await new Promise(r => setTimeout(r, 500))
assert.strictEqual(await File.isImmutable(cacheFile), false, 'cache.json must stay mutable')
assert.strictEqual(errors.length, 0)
} finally {
close()
await cleanupRoot(dir)
}
})
it('deduplicates multiple fs events for the same datetime', async () => {
const dir = await mkTmp()
const vmDir = path.join(dir, 'vm')
await fs.mkdir(vmDir, { recursive: true })
let lockBackupCallCount = 0
const errors: unknown[] = []
const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 })
try {
const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`)
// Write the json file twice to trigger two fs events
await fs.writeFile(jsonFile, '{}')
await fs.writeFile(jsonFile, '{}')
await waitFor(() => File.isImmutable(jsonFile))
// The file should be immutable exactly once — lockBackup was called once
assert.strictEqual(await File.isImmutable(jsonFile), true)
assert.strictEqual(errors.length, 0)
} finally {
close()
await cleanupRoot(dir)
}
})
it('locks a flat VHD file when .json is written', async () => {
const dir = await mkTmp()
const vmDir = path.join(dir, 'vm')
const vdiDir = path.join(vmDir, 'vdis', JOB_UUID, VDI_UUID)
await fs.mkdir(vdiDir, { recursive: true })
const errors: unknown[] = []
const close = watchVmDirectory(vmDir, err => errors.push(err), { delayBetweenSizeCheck: 100 })
try {
const vhdFile = path.join(vdiDir, `${BACKUP_DATE}.vhd`)
const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`)
await fs.writeFile(vhdFile, 'fake vhd data')
await fs.writeFile(jsonFile, '{}')
await waitFor(() => File.isImmutable(vhdFile))
assert.strictEqual(await File.isImmutable(vhdFile), true, 'flat .vhd must be immutable')
assert.strictEqual(errors.length, 0)
} finally {
close()
await cleanupRoot(dir)
}
})
})

386
@xen-orchestra/immutable-backups/src/_watcher.mts Normal file
View File

@@ -0,0 +1,386 @@
import fs from 'node:fs'
import fsp from 'node:fs/promises'
import { join } from 'node:path'
import * as Directory from './directory.mjs'
import { createLogger } from '@xen-orchestra/log'
const { debug } = createLogger('xen-orchestra:immutable-backups:watcher')
export interface WatchOptions {
/** Milliseconds after which a datetime is evicted from the deduplication set inside watchVmDirectory. */
lockTimeout?: number
/** Milliseconds to wait between consecutive size checks when polling for write completion. */
delayBetweenSizeCheck: number
}
/**
* Matches the datetime prefix shared by all files belonging to a single backup run.
*
* Examples:
* "20231215T142030.json" → "20231215T142030"
* "20231215T142030.alias.vhd" → "20231215T142030"
* "cache.json.gz" → undefined (not a backup file)
*/
const DATETIME_RE = /^(\d{8}T\d{6}Z?)\./
function extractDatetime(filename: string): string | undefined {
return DATETIME_RE.exec(filename)?.[1]
}
// Poll `path` with `fs.stat` until the file size stops changing, indicating the
// write is complete regardless of file format (plain or encrypted).
// The first stat is issued immediately; resolution requires a stable non-zero
// size across two consecutive polls spaced 100 ms apart.
// Rejects with an error if `timeout` ms elapse before stability is reached.
export async function waitForWriteDone(path: string, timeout: number, delayBetweenSizeCheck: number): Promise<void> {
const deadline = Date.now() + timeout
let prevSize = -1
while (Date.now() < deadline) {
try {
const { size } = await fsp.stat(path)
if (size > 0 && size === prevSize) {
return // non-empty and size stable — write done
}
prevSize = size
} catch (err) {
if (err.code !== 'ENOENT') {
throw err
}
// file not yet visible — keep waiting
}
await new Promise<void>(resolve => setTimeout(resolve, delayBetweenSizeCheck))
}
throw new Error(`Timeout waiting for write to complete on ${path}`)
}
// Lock every file belonging to the backup run identified by `datetime` inside `vmDir`.
// Called once the terminal `.json` file is fully written, which guarantees all other
// files for that run are already fully written.
//
// All candidates — flat files (json, xva, checksum, plain/alias VHDs) and VHD
// directories — are passed to a single `chattr +i -R` invocation. For regular
// files `-R` is a no-op; missing optional paths are silently ignored by
// Directory.makeImmutableBatch.
async function lockBackup(vmDir: string, datetime: string): Promise<void> {
debug(`[watcher] lockBackup: vmDir="${vmDir}" datetime="${datetime}"`)
const candidates: string[] = [
join(vmDir, `${datetime}.json`),
join(vmDir, `${datetime}.xva`),
join(vmDir, `${datetime}.xva.checksum`),
]
const vdisDir = join(vmDir, 'vdis')
let jobIds: string[] = []
try {
jobIds = await fsp.readdir(vdisDir)
} catch (err) {
if (err.code !== 'ENOENT') throw err
}
await Promise.all(
jobIds.map(async jobId => {
let vdiIds: string[]
try {
vdiIds = await fsp.readdir(join(vdisDir, jobId))
} catch {
// full backups won't have any vdi and it's ok
return
}
for (const vdiId of vdiIds) {
const vdiDir = join(vdisDir, jobId, vdiId)
candidates.push(join(vdiDir, `${datetime}.vhd`))
candidates.push(join(vdiDir, `${datetime}.alias.vhd`))
candidates.push(join(vdiDir, 'data', `${datetime}.vhd`))
}
})
)
await Directory.makeImmutableBatch(candidates)
debug(`[watcher] lockBackup: done vmDir="${vmDir}" datetime="${datetime}"`)
}
// Watch a single VM backup directory (`xo-vm-backups/<VM UUID>/`) for newly
// completed backups and lock all their files.
//
// Trigger: a `<YYYYMMDD>T<HHmmss>.json` file appearing as a direct child whose
// size is stable (write complete). The `.json` is always written last, so its
// stability guarantees that every other file for that run is fully written.
//
// Returns a close function that stops the watcher.
export function watchVmDirectory(
vmDir: string,
onError: (err: unknown) => void,
{ lockTimeout = 10 * 60 * 1000, delayBetweenSizeCheck }: WatchOptions
): () => void {
const watcher = fs.watch(vmDir)
// Deduplicates lock attempts when multiple fs events fire for the same datetime.
// Entries are evicted after `lockTimeout` ms to keep the set bounded.
const lockedDatetimes = new Set<string>()
watcher.on('change', (_eventType: string, filename: string | Buffer | null) => {
if (filename == null) {
return
}
const name = String(filename)
debug(`[watcher] watchVmDirectory(${vmDir}): event filename="${name}"`)
if (!name.endsWith('.json')) {
return
}
const datetime = extractDatetime(name)
debug(`[watcher] watchVmDirectory: extractDatetime("${name}") → ${datetime}`)
if (datetime === undefined) {
debug(`[watcher] watchVmDirectory: ignoring "${name}" — does not match DATETIME_RE ${DATETIME_RE}`)
return // e.g. cache.json.gz
}
// ensure we handle event once per file
// clear up the remaingin data after a reasonnable tie
if (lockedDatetimes.has(datetime)) {
return
}
lockedDatetimes.add(datetime)
setTimeout(() => {
lockedDatetimes.delete(datetime)
}, lockTimeout).unref()
const jsonPath = join(vmDir, name)
waitForWriteDone(jsonPath, lockTimeout, delayBetweenSizeCheck)
.then(() => {
debug(`[watcher] watchVmDirectory: locking backup datetime="${datetime}" in vmDir="${vmDir}"`)
return lockBackup(vmDir, datetime)
})
.catch(err => {
const code = err.code
if (code === 'ENOENT') {
// file disappeared before write completed — next fs event will retry
debug(`[watcher] watchVmDirectory: file gone (ENOENT) — will retry on next event`)
return
}
onError(err)
})
})
watcher.on('error', onError)
return () => {
watcher.close()
watcher.removeAllListeners()
}
}
// Watch a backup date directory (`<YYYYMMDD>T<HHmmss>/`) for its terminal
// `metadata.json`. When its write is complete, all files in the directory are
// made immutable.
//
// Applies to both config backups and pool metadata backups: `metadata.json`
// is always written last.
//
// Returns a close function.
function watchBackupDateDirectory(
dateDir: string,
onError: (err: unknown) => void,
{ lockTimeout = 10 * 60 * 1000, delayBetweenSizeCheck }: WatchOptions
): () => void {
const watcher = fs.watch(dateDir)
let locked = false
function close() {
watcher.close()
watcher.removeAllListeners()
}
watcher.on('change', (_eventType: string, filename: string | Buffer | null) => {
if (filename == null || String(filename) !== 'metadata.json') {
return
}
if (locked) {
return
}
const metadataPath = join(dateDir, 'metadata.json')
waitForWriteDone(metadataPath, lockTimeout, delayBetweenSizeCheck)
.then(() => {
if (locked) {
return
}
locked = true
return Directory.makeImmutable(dateDir).then(close)
})
.catch(err => {
// we won't retry on error tot ensure we don't leave watcher dangling
close()
const code = err.code
if (code === 'ENOENT') {
return // maybe an incomplete write that has been purged
}
onError(err)
})
})
watcher.on('error', err => {
close()
onError(err)
})
return close
}
// Watch `dir` for subdirectories, invoke `makeChildWatcher` for each one,
// and close the child watcher when the subdirectory is removed.
//
// `makeChildWatcher` may return a close function or a Promise that resolves to one.
async function watchSubdirectoriesWithChildren(
dir: string,
makeChildWatcher: (subdir: string) => (() => void) | Promise<() => void>,
onError: (err: unknown) => void
): Promise<() => void> {
const childClosers = new Map<string, Promise<() => void>>()
const closeWatcher = await watchSubdirectories(
dir,
{
onAdd: subdir => {
if (!childClosers.has(subdir)) {
childClosers.set(
subdir,
Promise.resolve(makeChildWatcher(subdir)).catch(err => {
onError(err)
return () => {}
})
)
}
},
onRemove: subdir => {
const p = childClosers.get(subdir)
childClosers.delete(subdir)
p?.then(close => close()).catch(onError)
},
},
onError
)
return () => {
closeWatcher()
for (const p of childClosers.values()) {
p.then(close => close()).catch(onError)
}
childClosers.clear()
}
}
// Watch `dir` for immediate subdirectory additions and deletions. Creates `dir`
// if it does not exist. Calls `onAdd` for each existing and new subdirectory,
// and `onRemove` when one disappears.
async function watchSubdirectories(
dir: string,
{ onAdd, onRemove }: { onAdd: (subdir: string) => void; onRemove: (subdir: string) => void },
onError: (err: unknown) => void
): Promise<() => void> {
await fsp.mkdir(dir, { recursive: true })
const watcher = fs.watch(dir)
function close() {
watcher.close()
watcher.removeAllListeners()
}
watcher.on('change', (_eventType: string, filename: string | Buffer | null) => {
if (filename == null) {
return
}
const subdir = join(dir, String(filename))
debug(`[watcher] watchSubdirectories(${dir}): event filename="${String(filename)}"`)
fsp.stat(subdir).then(
stat => {
if (stat.isDirectory()) {
debug(`[watcher] watchSubdirectories: new dir detected "${subdir}"`)
onAdd(subdir)
} else {
debug(`[watcher] watchSubdirectories: "${subdir}" is not a directory (isFile=${stat.isFile()}) — ignored`)
}
},
(err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') {
onRemove(subdir)
return
}
onError(err)
}
)
})
watcher.on('error', err => {
close()
onError(err)
})
const entries = await fsp.readdir(dir, { withFileTypes: true })
for (const entry of entries) {
if (entry.isDirectory()) {
onAdd(join(dir, entry.name))
}
}
return close
}
// Watch a backup remote root and lock all completed backup files.
//
// Handles three backup types:
// - VM backups (`xo-vm-backups/<vmUUID>/`) — triggered by `<YYYYMMDD>T<HHmmss>.json`
// - Config backups (`xo-config-backups/<scheduleId>/<YYYYMMDD>T<HHmmss>/`) — triggered by `metadata.json`
// - Pool metadata (`xo-pool-metadata-backups/<scheduleId>/<poolUUID>/<YYYYMMDD>T<HHmmss>/`) — triggered by `metadata.json`
//
// Returns a close function.
export async function watchRemote(
root: string,
onError: (err: unknown) => void,
options: WatchOptions
): Promise<() => void> {
const [closeVmWatcher, closeConfigWatcher, closePoolWatcher] = await Promise.all([
// xo-vm-backups/<vmUUID>/
watchSubdirectoriesWithChildren(
join(root, 'xo-vm-backups'),
vmDir => watchVmDirectory(vmDir, onError, options),
onError
),
// xo-config-backups/<scheduleId>/<YYYYMMDD>T<HHmmss>/
watchSubdirectoriesWithChildren(
join(root, 'xo-config-backups'),
scheduleDir =>
watchSubdirectoriesWithChildren(
scheduleDir,
dateDir => watchBackupDateDirectory(dateDir, onError, options),
onError
),
onError
),
// xo-pool-metadata-backups/<scheduleId>/<poolUUID>/<YYYYMMDD>T<HHmmss>/
watchSubdirectoriesWithChildren(
join(root, 'xo-pool-metadata-backups'),
scheduleDir =>
watchSubdirectoriesWithChildren(
scheduleDir,
poolDir =>
watchSubdirectoriesWithChildren(
poolDir,
dateDir => watchBackupDateDirectory(dateDir, onError, options),
onError
),
onError
),
onError
),
])
return () => {
closeVmWatcher()
closeConfigWatcher()
closePoolWatcher()
}
}

4
@xen-orchestra/immutable-backups/src/declarations.d.ts vendored Normal file
View File

@@ -0,0 +1,4 @@
declare module 'app-conf' {
function load(name: string, opts?: { appDir?: string; ignoreUnknownFormats?: boolean }): Promise<any>
export { load }
}

95
@xen-orchestra/immutable-backups/src/directory.integ.mts Normal file
View File

@@ -0,0 +1,95 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import * as Directory from './directory.mjs'
import { tmpdir } from 'node:os'
import { rimraf } from 'rimraf'
describe('immutable-backups/file', async () => {
it('really lock a directory', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
try {
const dataDir = path.join(dir, 'data')
await fs.mkdir(dataDir)
const filePath = path.join(dataDir, 'test')
await fs.writeFile(filePath, 'data')
await Directory.makeImmutable(dataDir)
assert.strictEqual(await Directory.isImmutable(dataDir), true)
await assert.rejects(() => fs.writeFile(filePath, 'data'))
await assert.rejects(() => fs.appendFile(filePath, 'data'))
await assert.rejects(() => fs.unlink(filePath))
await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data'))
await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy'))
await Directory.liftImmutability(dataDir)
assert.strictEqual(await Directory.isImmutable(dataDir), false)
await fs.writeFile(filePath, 'data')
await fs.appendFile(filePath, 'data')
await fs.unlink(filePath)
await fs.rename(dataDir, dataDir + 'copy')
} finally {
await Directory.liftImmutabilityBatch([dir]).catch(() => {})
await rimraf(dir)
}
})
it('makeImmutableBatch locks existing files and directories, ignores missing paths', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
try {
const fileA = path.join(dir, 'a.txt')
const fileB = path.join(dir, 'b.txt')
const subDir = path.join(dir, 'sub')
const subFile = path.join(subDir, 'c.txt')
const missing = path.join(dir, 'does-not-exist')
await fs.writeFile(fileA, 'aaa')
await fs.writeFile(fileB, 'bbb')
await fs.mkdir(subDir)
await fs.writeFile(subFile, 'ccc')
// missing path must not throw
await assert.doesNotReject(() => Directory.makeImmutableBatch([fileA, fileB, subDir, missing]))
assert.strictEqual(await Directory.isImmutable(fileA), true, 'fileA should be immutable')
assert.strictEqual(await Directory.isImmutable(fileB), true, 'fileB should be immutable')
assert.strictEqual(await Directory.isImmutable(subDir), true, 'subDir should be immutable')
assert.strictEqual(await Directory.isImmutable(subFile), true, 'subFile should be immutable (recursive)')
await assert.rejects(() => fs.writeFile(fileA, 'tampered'), { code: 'EPERM' })
await assert.rejects(() => fs.writeFile(subFile, 'tampered'), { code: 'EPERM' })
} finally {
// lift before rimraf — rimraf cannot delete immutable files
await Directory.liftImmutabilityBatch([dir]).catch(() => {})
await rimraf(dir)
}
})
it('liftImmutabilityBatch unlocks existing files and directories, ignores missing paths', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
try {
const fileA = path.join(dir, 'a.txt')
const subDir = path.join(dir, 'sub')
const subFile = path.join(subDir, 'c.txt')
const missing = path.join(dir, 'does-not-exist')
await fs.writeFile(fileA, 'aaa')
await fs.mkdir(subDir)
await fs.writeFile(subFile, 'ccc')
await Directory.makeImmutableBatch([fileA, subDir])
// missing path must not throw
await assert.doesNotReject(() => Directory.liftImmutabilityBatch([fileA, subDir, missing]))
assert.strictEqual(await Directory.isImmutable(fileA), false, 'fileA should be mutable again')
assert.strictEqual(await Directory.isImmutable(subDir), false, 'subDir should be mutable again')
assert.strictEqual(await Directory.isImmutable(subFile), false, 'subFile should be mutable again')
await fs.writeFile(fileA, 'updated')
await fs.writeFile(subFile, 'updated')
} finally {
await Directory.liftImmutabilityBatch([dir]).catch(() => {})
await rimraf(dir)
}
})
})

50
@xen-orchestra/immutable-backups/src/directory.mts Normal file
View File

@@ -0,0 +1,50 @@
import execa from 'execa'
// Recursively set the immutable (`+i`) attribute on a directory and all its contents.
export async function makeImmutable(dirPath: string): Promise<void> {
await execa('chattr', ['+i', '-R', dirPath])
}
// chattr processes all paths even when some are missing (it does not abort on the first
// error), so every existing path is correctly lifted. Per-path "No such file or
// directory while trying to stat" messages are silently ignored; any other error
// (e.g. permission denied) causes the error to be re-thrown.
async function execChattrWithMissingFiles(args: string[]) {
try {
await execa('chattr', args)
} catch (err) {
const stderr: string = 'stderr' in err ? err.stderr : ''
const hasUnexpected = stderr
.split('\n')
.some(line => line.trim() !== '' && !line.includes('No such file or directory while trying to stat'))
if (hasUnexpected) {
throw err
}
}
}
// Lock multiple paths (files and/or directories) with a single `chattr +i -R` invocation.
// For regular files `-R` is a no-op (chattr ignores it). Missing paths are silently
// ignored: chattr processes all remaining paths before exiting non-zero.
export async function makeImmutableBatch(paths: string[]): Promise<void> {
if (paths.length === 0) return
await execChattrWithMissingFiles(['+i', '-R', ...paths])
}
// Recursively remove the immutable (`-i`) attribute from a directory and all its contents.
export async function liftImmutability(dirPath: string): Promise<void> {
await execa('chattr', ['-i', '-R', dirPath])
}
// Lift immutability from multiple paths with a single `chattr -i -R` invocation.
// Works for both flat files and directories.
export async function liftImmutabilityBatch(paths: string[]): Promise<void> {
if (paths.length === 0) return
await execChattrWithMissingFiles(['-i', '-R', ...paths])
}
// Returns whether the immutable (`i`) attribute is set on `path`.
export async function isImmutable(path: string): Promise<boolean> {
const { stdout } = await execa('lsattr', ['-d', path])
return stdout[4] === 'i'
}

5
@xen-orchestra/immutable-backups/file.integ.mjs → @xen-orchestra/immutable-backups/src/file.integ.mts
View File

@@ -9,17 +9,16 @@ import { rimraf } from 'rimraf'
describe('immutable-backups/file', async () => {
it('really lock a file', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
const immutDir = path.join(dir, '.immutable')
const filePath = path.join(dir, 'test.ext')
await fs.writeFile(filePath, 'data')
assert.strictEqual(await File.isImmutable(filePath), false)
await File.makeImmutable(filePath, immutDir)
await File.makeImmutable(filePath)
assert.strictEqual(await File.isImmutable(filePath), true)
await assert.rejects(() => fs.writeFile(filePath, 'data'))
await assert.rejects(() => fs.appendFile(filePath, 'data'))
await assert.rejects(() => fs.unlink(filePath))
await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
await File.liftImmutability(filePath, immutDir)
await File.liftImmutability(filePath)
assert.strictEqual(await File.isImmutable(filePath), false)
await fs.writeFile(filePath, 'data')
await fs.appendFile(filePath, 'data')

20
@xen-orchestra/immutable-backups/src/file.mts Normal file
View File

@@ -0,0 +1,20 @@
import execa from 'execa'
// this work only on linux like systems
// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/
// Set the immutable (`+i`) attribute on a single file.
export async function makeImmutable(path: string): Promise<void> {
await execa('chattr', ['+i', path])
}
// Remove the immutable (`-i`) attribute from a single file.
export async function liftImmutability(filePath: string): Promise<void> {
await execa('chattr', ['-i', filePath])
}
// Returns whether the immutable (`i`) attribute is set on `path`.
export async function isImmutable(path: string): Promise<boolean> {
const { stdout } = await execa('lsattr', ['-d', path])
return stdout[4] === 'i'
}

2
@xen-orchestra/immutable-backups/src/isBackupMetadata.mts Normal file
View File

@@ -0,0 +1,2 @@
// Returns whether `path` points to a VM backup metadata JSON file.
export default (path: string): RegExpMatchArray | null => path.match(/xo-vm-backups\/[^/]+\/[^/]+\.json$/)

183
@xen-orchestra/immutable-backups/src/liftProtection.mts Normal file
View File

@@ -0,0 +1,183 @@
import fsp from 'node:fs/promises'
import { join } from 'node:path'
import * as Directory from './directory.mjs'
import * as File from './file.mjs'
import { createLogger } from '@xen-orchestra/log'
import { asyncEach } from '@vates/async-each'
import cleanXoCache from './_cleanXoCache.mjs'
import { RemoteConfig } from './_loadConfig.mjs'
const { warn } = createLogger('xen-orchestra:immutable-backups:liftProtection')
/**
* Matches the datetime prefix shared by all files belonging to a single VM backup run.
*
* Examples:
* "20231215T142030.json" → "20231215T142030"
* "20231215T142030Z.alias.vhd" → "20231215T142030Z"
* "cache.json.gz" → undefined (not a backup file)
*/
const DATETIME_RE = /^(\d{8}T\d{6}Z?)\./
function extractDatetime(filename: string): string | undefined {
return DATETIME_RE.exec(filename)?.[1]
}
// Returns the absolute paths of all immediate subdirectories of `dir`.
// Returns [] if `dir` does not exist.
async function listDirs(dir: string): Promise<string[]> {
try {
const entries = await fsp.readdir(dir, { withFileTypes: true })
return entries.filter(e => e.isDirectory()).map(e => join(dir, e.name))
} catch (err) {
if (err.code === 'ENOENT') return []
throw err
}
}
// Lift immutability from every file and subdirectory inside a date-stamped
// backup directory (config backup or pool-metadata backup).
async function liftDirBackup(dateDir: string): Promise<void> {
const entries = await fsp.readdir(dateDir, { withFileTypes: true })
const paths = entries.map(entry => join(dateDir, entry.name))
await Directory.liftImmutabilityBatch(paths)
}
// Walk `xo-vm-backups/<vmUUID>/<datetime>.json` files and lift immutability on
// any VM backup run whose metadata mtime is older than `immutabilityDuration`.
// Per vmDir: vdis is read once, all expired datetimes are batched into a single
// liftImmutabilityBatch call, and cleanXoCache is called once.
async function liftExpiredVmBackups(root: string, immutabilityDuration: number): Promise<void> {
const threshold = Date.now() - immutabilityDuration
await asyncEach(await listDirs(join(root, 'xo-vm-backups')), async vmDir => {
// 1. Find all expired datetimes in this vmDir.
const entries = await fsp.readdir(vmDir, { withFileTypes: true }).catch(() => [])
const expiredDatetimes: string[] = []
let firstExpiredJsonPath: string | undefined
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith('.json')) continue
const datetime = extractDatetime(entry.name)
if (datetime === undefined) continue // e.g. cache.json.gz
const jsonPath = join(vmDir, entry.name)
try {
if (!(await File.isImmutable(jsonPath))) continue
const { mtimeMs } = await fsp.stat(jsonPath)
if (mtimeMs > threshold) continue
expiredDatetimes.push(datetime)
firstExpiredJsonPath ??= jsonPath
} catch (err) {
warn('error checking VM backup expiry', { err, jsonPath })
}
}
if (expiredDatetimes.length === 0) return
// 2. Read the vdis tree once for this vmDir.
const vdisDir = join(vmDir, 'vdis')
let jobIds: string[] = []
try {
jobIds = await fsp.readdir(vdisDir)
} catch (err) {
if (err.code !== 'ENOENT') throw err
}
const vdiDirs: string[] = []
await asyncEach(jobIds, async jobId => {
let vdiIds: string[]
try {
vdiIds = await fsp.readdir(join(vdisDir, jobId))
} catch {
return
}
for (const vdiId of vdiIds) {
vdiDirs.push(join(vdisDir, jobId, vdiId))
}
})
// 3. Build all candidates for all expired datetimes in one pass.
const candidates: string[] = []
for (const datetime of expiredDatetimes) {
candidates.push(
join(vmDir, `${datetime}.json`),
join(vmDir, `${datetime}.xva`),
join(vmDir, `${datetime}.xva.checksum`)
)
for (const vdiDir of vdiDirs) {
candidates.push(
join(vdiDir, `${datetime}.vhd`),
join(vdiDir, `${datetime}.alias.vhd`),
join(vdiDir, 'data', `${datetime}.vhd`)
)
}
}
// 4. Single batch lift + single cache invalidation for this vmDir.
try {
await Directory.liftImmutabilityBatch(candidates)
await cleanXoCache(firstExpiredJsonPath!)
} catch (err) {
warn('error lifting VM backup immutability', { err, vmDir })
}
})
}
// Walk `xo-config-backups/<scheduleId>/<datetime>/metadata.json` files and
// lift immutability on any backup directory whose metadata mtime is expired.
async function liftExpiredConfigBackups(root: string, immutabilityDuration: number): Promise<void> {
const threshold = Date.now() - immutabilityDuration
await asyncEach(await listDirs(join(root, 'xo-config-backups')), async scheduleDir => {
for (const dateDir of await listDirs(scheduleDir)) {
const metadataPath = join(dateDir, 'metadata.json')
try {
if (!(await File.isImmutable(metadataPath))) continue
const { mtimeMs } = await fsp.stat(metadataPath)
if (mtimeMs > threshold) continue
await liftDirBackup(dateDir)
} catch (err) {
const code = err.code
if (code !== 'ENOENT') warn('error lifting config backup immutability', { err, metadataPath })
}
}
})
}
// Walk `xo-pool-metadata-backups/<scheduleId>/<poolUUID>/<datetime>/metadata.json`
// files and lift immutability on any backup directory whose metadata mtime is expired.
async function liftExpiredPoolBackups(root: string, immutabilityDuration: number): Promise<void> {
const threshold = Date.now() - immutabilityDuration
await asyncEach(await listDirs(join(root, 'xo-pool-metadata-backups')), async scheduleDir => {
for (const poolDir of await listDirs(scheduleDir)) {
for (const dateDir of await listDirs(poolDir)) {
const metadataPath = join(dateDir, 'metadata.json')
try {
if (!(await File.isImmutable(metadataPath))) continue
const { mtimeMs } = await fsp.stat(metadataPath)
if (mtimeMs > threshold) continue
await liftDirBackup(dateDir)
} catch (err) {
const code = err.code
if (code !== 'ENOENT') warn('error lifting pool metadata backup immutability', { err, metadataPath })
}
}
}
})
}
// Scan the filesystem for expired immutable backups under `root` and lift their
// immutability. No index is required — the backup tree is walked directly.
export async function liftRemoteImmutability(root: string, immutabilityDuration: number): Promise<void> {
await Promise.all([
liftExpiredVmBackups(root, immutabilityDuration),
liftExpiredConfigBackups(root, immutabilityDuration),
liftExpiredPoolBackups(root, immutabilityDuration),
])
}
// Lift immutability on all expired backups across every configured remote.
export async function liftImmutability(remotes: Record<string, RemoteConfig>): Promise<void> {
for (const [remoteId, { root, immutabilityDuration }] of Object.entries(remotes)) {
await liftRemoteImmutability(root, immutabilityDuration).catch(err =>
warn('error during liftRemoteImmutability', { err, remoteId, root, immutabilityDuration })
)
}
}

18
@xen-orchestra/immutable-backups/src/protectRemotes.mts Normal file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env node
import { createLogger } from '@xen-orchestra/log'
import loadConfig from './_loadConfig.mjs'
import { watchRemote } from './remote.mjs'
import { liftImmutability } from './liftProtection.mjs'
const { info, warn } = createLogger('xen-orchestra:immutable-backups:remote')
const { liftEvery, remotes } = await loadConfig()
for (const [remoteId, remote] of Object.entries(remotes)) {
watchRemote(remoteId, remote).catch(err => warn('error during watchRemote', { err, remoteId, remote }))
}
info('setup watcher for immutability lifting')
setInterval(async () => {
await liftImmutability(remotes).catch(error => warn('error while lifting immutability', error))
}, liftEvery)

241
@xen-orchestra/immutable-backups/src/remote.integ.mts Normal file
View File

@@ -0,0 +1,241 @@
import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import { tmpdir } from 'node:os'
import { rimraf } from 'rimraf'
import execa from 'execa'
import * as File from './file.mjs'
import * as Directory from './directory.mjs'
import { watchRemote } from './remote.mjs'
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const ONE_DAY_MS = 24 * 60 * 60 * 1000
// Fake UUIDs used across all tests
const VM_UUID = 'aaaaaaaa-0000-0000-0000-000000000001'
const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001'
const VDI_UUID = 'cccccccc-0000-0000-0000-000000000001'
const SCHEDULE_UUID = 'dddddddd-0000-0000-0000-000000000001'
const BACKUP_DATE = '20240115T120000Z'
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
// Poll `fn` every 200 ms until it returns a truthy value or `timeout` ms
// have elapsed. Throws if the deadline is reached.
async function waitFor(fn: () => Promise<boolean>, timeout = 8000): Promise<void> {
const deadline = Date.now() + timeout
while (Date.now() < deadline) {
try {
if (await fn()) return
} catch {}
await new Promise(resolve => setTimeout(resolve, 200))
}
throw new Error('Timed out waiting for condition')
}
// Recursively lift all immutable flags inside `root`, then delete the tree.
// Must be run as root since chattr requires elevated privileges.
async function cleanupRoot(root: string): Promise<void> {
try {
// -i removes the immutable flag; -R recurses into every file and directory.
await execa('chattr', ['-i', '-R', root])
} catch (_err) {}
await rimraf(root)
}
// Start a watcher on a fresh temp directory and return both the root path and
// the index path so the caller can create files and assert on them.
//
// `preDirs` is a list of relative paths that will be created inside `root`
// **before** starting the watcher. The watcher must see these directories at
// startup so it can detect new files written into them later.
async function makeRemote(preDirs: string[] = []): Promise<{ root: string; close: () => void }> {
const root = await fs.mkdtemp(path.join(tmpdir(), 'immut-test-'))
// Pre-create directories so the watcher has them in its watch list at startup.
for (const relDir of preDirs) {
await fs.mkdir(path.join(root, relDir), { recursive: true })
}
const { close } = await watchRemote('test', { root, immutabilityDuration: ONE_DAY_MS, delayBetweenSizeCheck: 100 })
// Give the watcher a moment to complete its initial scan and become ready before
// the caller starts writing files.
await new Promise(resolve => setTimeout(resolve, 500))
return { root, close }
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('protectRemotes/watchRemote', async () => {
it('makes VM backup flat files (.json / .xva / .xva.checksum) immutable', async () => {
const vmRelDir = path.join('xo-vm-backups', VM_UUID)
const { root, close } = await makeRemote([vmRelDir])
try {
const vmDir = path.join(root, vmRelDir)
const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`)
const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`)
const checksumFile = path.join(vmDir, `${BACKUP_DATE}.xva.checksum`)
// This file does NOT match any watched glob — it must stay mutable.
const ignoredFile = path.join(vmDir, 'not-a-backup.txt')
// xva and checksum must exist before json — json is the terminal signal
await fs.writeFile(xvaFile, 'fake xva data')
await fs.writeFile(checksumFile, 'abc123')
await fs.writeFile(ignoredFile, 'should stay mutable')
await fs.writeFile(jsonFile, '{}')
await waitFor(() => File.isImmutable(jsonFile))
await waitFor(() => File.isImmutable(xvaFile))
await waitFor(() => File.isImmutable(checksumFile))
assert.strictEqual(await File.isImmutable(jsonFile), true, '.json should be immutable')
assert.strictEqual(await File.isImmutable(xvaFile), true, '.xva should be immutable')
assert.strictEqual(await File.isImmutable(checksumFile), true, '.xva.checksum should be immutable')
// Confirm writes and deletes are actually blocked.
await assert.rejects(fs.writeFile(jsonFile, 'tampered'), { code: 'EPERM' })
await assert.rejects(fs.unlink(xvaFile), { code: 'EPERM' })
// Files outside the watched patterns must remain mutable.
assert.strictEqual(await File.isImmutable(ignoredFile), false, 'non-backup file must stay mutable')
} finally {
close()
await cleanupRoot(root)
}
})
it('makes a VHD directory immutable once the json is written', async () => {
const vmRelDir = path.join('xo-vm-backups', VM_UUID)
const dataRelDir = path.join(vmRelDir, 'vdis', JOB_UUID, VDI_UUID, 'data')
const { root, close } = await makeRemote([dataRelDir])
try {
// xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/data/<date>.vhd/
const vmDir = path.join(root, vmRelDir)
const vhdDir = path.join(root, dataRelDir, `${BACKUP_DATE}.vhd`)
await fs.mkdir(vhdDir, { recursive: true })
const bat = path.join(vhdDir, 'bat')
const header = path.join(vhdDir, 'header')
const footer = path.join(vhdDir, 'footer')
// Subdirectory inside the VHD dir that is NOT a watched key file.
const blocks = path.join(vhdDir, 'blocks')
await fs.mkdir(blocks)
await fs.writeFile(bat, 'bat data')
await fs.writeFile(header, 'header data')
await fs.writeFile(footer, 'footer data')
// json written last — triggers lockBackup which locks the VHD directory
await fs.writeFile(path.join(vmDir, `${BACKUP_DATE}.json`), '{}')
// The whole directory (and its contents) must become immutable.
await waitFor(() => Directory.isImmutable(vhdDir))
assert.strictEqual(await Directory.isImmutable(vhdDir), true, 'vhd dir should be immutable')
assert.strictEqual(await File.isImmutable(bat), true, 'bat should be immutable')
assert.strictEqual(await File.isImmutable(header), true, 'header should be immutable')
assert.strictEqual(await File.isImmutable(footer), true, 'footer should be immutable')
// Confirm the lock is real.
await assert.rejects(fs.writeFile(bat, 'tampered'), { code: 'EPERM' })
await assert.rejects(fs.mkdir(path.join(vhdDir, 'new-subdir')), { code: 'EPERM' })
} finally {
close()
await cleanupRoot(root)
}
})
it('does NOT lock the VHD directory when the metadata are not written', async () => {
const dataRelDir = path.join('xo-vm-backups', VM_UUID, 'vdis', JOB_UUID, VDI_UUID, 'data')
const { root, close } = await makeRemote([dataRelDir])
try {
const vhdDir = path.join(root, dataRelDir, `${BACKUP_DATE}.vhd`)
await fs.mkdir(vhdDir, { recursive: true })
const bat = path.join(vhdDir, 'bat')
const header = path.join(vhdDir, 'header')
// footer is intentionally omitted.
await fs.writeFile(bat, 'bat data')
await fs.writeFile(header, 'header data')
// Wait longer than awaitWriteFinish (2 s) + processing to be certain
// the watcher has handled both events without locking anything.
await new Promise(resolve => setTimeout(resolve, 5000))
assert.strictEqual(await Directory.isImmutable(vhdDir), false, 'dir must stay mutable')
assert.strictEqual(await File.isImmutable(bat), false, 'bat must stay mutable')
assert.strictEqual(await File.isImmutable(header), false, 'header must stay mutable')
} finally {
close()
await cleanupRoot(root)
}
})
it('makes config-backup files immutable', async () => {
const scheduleRelDir = path.join('xo-config-backups', SCHEDULE_UUID)
const { root, close } = await makeRemote([scheduleRelDir])
try {
// xo-config-backups/<scheduleId>/<date>/
const configDir = path.join(root, scheduleRelDir, BACKUP_DATE)
await fs.mkdir(configDir, { recursive: true })
const dataFile = path.join(configDir, 'data')
const dataJsonFile = path.join(configDir, 'data.json')
const metadataFile = path.join(configDir, 'metadata.json')
await fs.writeFile(dataFile, 'config backup data')
await fs.writeFile(dataJsonFile, '{}')
await fs.writeFile(metadataFile, '{}')
await waitFor(() => File.isImmutable(dataFile))
await waitFor(() => File.isImmutable(dataJsonFile))
await waitFor(() => File.isImmutable(metadataFile))
assert.strictEqual(await File.isImmutable(dataFile), true, 'data should be immutable')
assert.strictEqual(await File.isImmutable(dataJsonFile), true, 'data.json should be immutable')
assert.strictEqual(await File.isImmutable(metadataFile), true, 'metadata.json should be immutable')
} finally {
close()
await cleanupRoot(root)
}
})
it('detects a new VM directory created after the watcher starts', async () => {
// Start with NO pre-existing VM directories.
// The watcher must dynamically pick up xo-vm-backups/<UUID>/ created
// after it is already running — this is the core of the cascading design.
const { root, close } = await makeRemote([])
try {
const newVmUuid = 'eeeeeeee-0000-0000-0000-000000000099'
const vmDir = path.join(root, 'xo-vm-backups', newVmUuid)
// Create the VM directory and write backup files AFTER the watcher is running.
await fs.mkdir(vmDir, { recursive: true })
// xva must be written before json (json is the terminal signal).
const xvaFile = path.join(vmDir, `${BACKUP_DATE}.xva`)
const jsonFile = path.join(vmDir, `${BACKUP_DATE}.json`)
await fs.writeFile(xvaFile, 'fake xva data')
await fs.writeFile(jsonFile, '{}')
await waitFor(() => File.isImmutable(jsonFile))
await waitFor(() => File.isImmutable(xvaFile))
assert.strictEqual(await File.isImmutable(jsonFile), true, '.json in new VM dir should be immutable')
assert.strictEqual(await File.isImmutable(xvaFile), true, '.xva in new VM dir should be immutable')
await assert.rejects(fs.writeFile(jsonFile, 'tampered'), { code: 'EPERM' })
} finally {
close()
await cleanupRoot(root)
}
})
})

461
@xen-orchestra/immutable-backups/src/remote.load.mts Normal file
View File

@@ -0,0 +1,461 @@
#!/usr/bin/env node
//
// Load test: 5000 VMs × 4 flat VHDs.
//
// Phase A — live watcher:
// Write backups for all VMs while the watcher is running and measure how
// long it takes for every backup to become immutable.
//
// Phase B — lift:
// Lift immutability on all locked files and measure the duration.
//
// Must be run as root (chattr requires elevated privileges).
// Usage: node @xen-orchestra/immutable-backups/remote.load.mjs
import fs from 'node:fs/promises'
import path from 'node:path'
import { tmpdir } from 'node:os'
import { rimraf } from 'rimraf'
import execa from 'execa'
import { asyncEach } from '@vates/async-each'
import * as File from './file.mjs'
import { watchRemote } from './remote.mjs'
import { liftRemoteImmutability } from './liftProtection.mjs'
// ---------------------------------------------------------------------------
// Config
// ---------------------------------------------------------------------------
const VM_COUNT = 5000
const DISKS_PER_VM = 4
const JOB_UUID = 'bbbbbbbb-0000-0000-0000-000000000001'
/** Datetime used for the live backups (Phase B). */
const BACKUP_DATE = '20240115T120000Z'
const ONE_DAY_MS = 24 * 60 * 60 * 1000
/** Max concurrent backup file writes. */
const WRITE_CONCURRENCY = 100
/** Max files per lsattr call (avoids ARG_MAX). */
const LSATTR_BATCH = 500
/** Max time to wait for all backups to be locked (ms). */
const LOCK_TIMEOUT_MS = 120_000
// Each VM backup: 1 .json + DISKS_PER_VM plain .vhd files.
const FILES_PER_VM = 1 + DISKS_PER_VM
const EXPECTED_TOTAL = VM_COUNT * FILES_PER_VM
/**
* Indices into `vms[]` whose json→immutable latency is tracked individually
* to show per-backup timing under full concurrent load.
*/
const PROBE_VM_INDICES = [
0,
Math.floor(VM_COUNT / 4),
Math.floor(VM_COUNT / 2),
Math.floor((VM_COUNT * 3) / 4),
VM_COUNT - 1,
]
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makeUuid(i: number): string {
return `${String(i).padStart(8, '0')}-0000-0000-0000-000000000000`
}
async function cleanupRoot(root: string): Promise<void> {
try {
await execa('chattr', ['-i', '-R', root])
} catch {}
await rimraf(root)
}
// Spin-polls File.isImmutable every 10 ms until the file is immutable or
// `timeoutMs` elapses. Returns the wall-clock timestamp (ms) when locked,
// or null on timeout.
async function waitUntilImmutable(filePath: string, timeoutMs = 60_000): Promise<number | null> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if (await File.isImmutable(filePath).catch(() => false)) return Date.now()
await new Promise(r => setTimeout(r, 10))
}
return null
}
// Count how many `<date>.json` files across all VM dirs are currently immutable.
// Uses batched lsattr calls to avoid hitting ARG_MAX.
async function countImmutableJsonFiles(root: string, date: string, vmUuids: string[]): Promise<number> {
let count = 0
for (let i = 0; i < vmUuids.length; i += LSATTR_BATCH) {
const batch = vmUuids.slice(i, i + LSATTR_BATCH).map(uuid => path.join(root, 'xo-vm-backups', uuid, `${date}.json`))
try {
const { stdout } = await execa('lsattr', ['-d', ...batch])
count += stdout.split('\n').filter(line => line.length > 4 && line[4] === 'i').length
} catch (err) {
// lsattr exits non-zero when some files don't exist yet — parse stdout anyway
const out = (err as any).stdout as string | undefined
if (out) count += out.split('\n').filter((line: string) => line.length > 4 && line[4] === 'i').length
}
}
return count
}
// Poll `countImmutableJsonFiles` until it reaches `target` or `LOCK_TIMEOUT_MS`
// elapses. Logs progress on every change.
async function waitForImmutableCount(
root: string,
date: string,
vmUuids: string[],
target: number,
label: string
): Promise<{ elapsed: number; reached: boolean }> {
const tStart = Date.now()
let lastCount = await countImmutableJsonFiles(root, date, vmUuids)
console.log(` [${label}] starting at ${lastCount}/${target}`)
const deadline = Date.now() + LOCK_TIMEOUT_MS
while (Date.now() < deadline) {
sampleMemory()
const count = await countImmutableJsonFiles(root, date, vmUuids)
if (count !== lastCount) {
console.log(
` [${label}] immutable json files: ${count}/${target}` +
` | ${Date.now() - tStart} ms elapsed` +
` | peak RSS ${mb(peakRss)}`
)
lastCount = count
}
if (count >= target) {
return { elapsed: Date.now() - tStart, reached: true }
}
await new Promise(resolve => setTimeout(resolve, 500))
}
return { elapsed: Date.now() - tStart, reached: false }
}
// ---------------------------------------------------------------------------
// Raw cost benchmark
// ---------------------------------------------------------------------------
function average(arr: number[]): number {
return arr.reduce((a, b) => a + b, 0) / arr.length
}
// Measures the cost of the individual operations that `lockBackup` performs,
// with no watcher running and no concurrent load, so results reflect pure
// operation overhead with no queue contention.
//
// Reports:
// 1. chattr +i — single subprocess spawn
// 2. 5 concurrent chattr +i — one realistic backup (1 json + 4 vhds in parallel)
async function measureRawCosts(scratchDir: string): Promise<void> {
console.log('=== Baseline: raw lockBackup operation costs (no watcher, no load) ===')
const files = await Promise.all(
Array.from({ length: 5 }, (_, i) => {
const p = path.join(scratchDir, `_measure_${i}.json`)
return fs.writeFile(p, '{}').then(() => p)
})
)
// 1. chattr +i — single subprocess spawn, 5 sequential runs
const tChattr1: number[] = []
for (const f of files) {
const t = process.hrtime.bigint()
await execa('chattr', ['+i', f])
tChattr1.push(Number(process.hrtime.bigint() - t) / 1e6)
await execa('chattr', ['-i', f])
}
console.log(
` chattr +i single spawn: avg ${average(tChattr1).toFixed(1)} ms` +
` [${tChattr1.map(t => t.toFixed(1)).join(', ')} ms]`
)
// 2. 5 concurrent chattr +i — realistic per-backup case (1 json + 4 vhds)
const tChattr5: number[] = []
for (let run = 0; run < 5; run++) {
const t = process.hrtime.bigint()
await Promise.all(files.map(f => execa('chattr', ['+i', f])))
tChattr5.push(Number(process.hrtime.bigint() - t) / 1e6)
await Promise.all(files.map(f => execa('chattr', ['-i', f])))
}
console.log(
` 5x concurrent chattr +i: avg ${average(tChattr5).toFixed(1)} ms` +
` [${tChattr5.map(t => t.toFixed(1)).join(', ')} ms]`
)
await Promise.all(files.map(f => fs.unlink(f)))
console.log('')
}
// ---------------------------------------------------------------------------
// Memory tracking
// ---------------------------------------------------------------------------
let peakRss = 0
function sampleMemory(): number {
const { rss } = process.memoryUsage()
if (rss > peakRss) peakRss = rss
return rss
}
function mb(bytes: number): string {
return (bytes / 1024 / 1024).toFixed(1) + ' MB'
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main(): Promise<void> {
const root = await fs.mkdtemp(path.join(tmpdir(), 'immut-load-'))
console.log(`Root: ${root}`)
console.log(`VMs: ${VM_COUNT}`)
console.log(`Disks per VM: ${DISKS_PER_VM}`)
console.log(`Files per VM: ${FILES_PER_VM}`)
console.log(`Expected total: ${EXPECTED_TOTAL}`)
console.log('')
const vms = Array.from({ length: VM_COUNT }, (_, i) => ({
uuid: makeUuid(i),
vdiUuids: Array.from({ length: DISKS_PER_VM }, (_, j) => makeUuid(VM_COUNT + i * DISKS_PER_VM + j)),
}))
const vmUuids = vms.map(vm => vm.uuid)
// -------------------------------------------------------------------------
// 1. Create directory structure
// -------------------------------------------------------------------------
console.log('Creating directory structure...')
const tDirs = Date.now()
await asyncEach(
vms,
async vm => {
await Promise.all(
vm.vdiUuids.map(vdiUuid =>
fs.mkdir(path.join(root, 'xo-vm-backups', vm.uuid, 'vdis', JOB_UUID, vdiUuid), {
recursive: true,
})
)
)
},
{ concurrency: 200 }
)
console.log(`Done in ${Date.now() - tDirs} ms (${VM_COUNT * DISKS_PER_VM} leaf dirs)`)
console.log('')
// -------------------------------------------------------------------------
// 1.5. Baseline: measure raw operation costs (no watcher, no contention)
// -------------------------------------------------------------------------
await measureRawCosts(path.join(root, 'xo-vm-backups', vms[0].uuid))
// -------------------------------------------------------------------------
// 2. Start memory sampling + watchRemote
// -------------------------------------------------------------------------
const memSampler = setInterval(sampleMemory, 50)
sampleMemory()
const rssBeforeWatch = process.memoryUsage().rss
console.log(`Memory before watchRemote: ${mb(rssBeforeWatch)}`)
const tWatchStart = Date.now()
const { close } = await watchRemote('load-test', {
root,
immutabilityDuration: ONE_DAY_MS,
delayBetweenSizeCheck: 100,
})
const tWatcherReady = Date.now() - tWatchStart
await new Promise(resolve => setTimeout(resolve, 500))
sampleMemory()
const rssAfterWatch = process.memoryUsage().rss
console.log(`watchRemote ready in ${tWatcherReady} ms`)
console.log(`Memory after watchRemote: ${mb(rssAfterWatch)} (delta: ${mb(rssAfterWatch - rssBeforeWatch)})`)
console.log('')
// -------------------------------------------------------------------------
// Phase A — single-backup probe: watcher running, zero concurrent load.
// Measures json-written → immutable latency for 5 sequential backups on a
// dedicated probe VM (not in the main 5000).
// Together with the baseline this isolates: event delivery + readdir overhead
// = (Phase A latency) − (5× concurrent chattr baseline).
// -------------------------------------------------------------------------
console.log('=== Phase A: single-backup probe latency (watcher running, no load) ===')
// Use UUIDs that cannot collide with the main vms[] (all-f prefix).
const PROBE_VM_UUID = 'ffffffff-ffff-ffff-ffff-000000000001'
const probeVm = {
uuid: PROBE_VM_UUID,
vdiUuids: Array.from(
{ length: DISKS_PER_VM },
(_, j) => `ffffffff-ffff-ffff-ffff-${String(j + 2).padStart(12, '0')}`
),
}
const probeVmDir = path.join(root, 'xo-vm-backups', PROBE_VM_UUID)
for (const vdiUuid of probeVm.vdiUuids) {
await fs.mkdir(path.join(probeVmDir, 'vdis', JOB_UUID, vdiUuid), { recursive: true })
}
// Give the watcher time to detect and start watching the new VM dir.
await new Promise(resolve => setTimeout(resolve, 500))
const probeSingleTimes: number[] = []
for (let run = 0; run < 5; run++) {
const pDatetime = `2024011${run + 1}T120000Z`
await Promise.all(
probeVm.vdiUuids.map(vdiUuid =>
fs.writeFile(path.join(probeVmDir, 'vdis', JOB_UUID, vdiUuid, `${pDatetime}.vhd`), 'fake vhd data')
)
)
const jsonPath = path.join(probeVmDir, `${pDatetime}.json`)
const tJsonWritten = Date.now()
await fs.writeFile(jsonPath, '{}')
const tLocked = await waitUntilImmutable(jsonPath)
if (tLocked !== null) {
const latency = tLocked - tJsonWritten
probeSingleTimes.push(latency)
console.log(` run ${run + 1}: json→immutable ${latency} ms`)
} else {
console.log(` run ${run + 1}: TIMEOUT`)
}
}
if (probeSingleTimes.length > 0) {
const sorted = [...probeSingleTimes].sort((a, b) => a - b)
const avg = average(probeSingleTimes)
const med = sorted[Math.floor(sorted.length / 2)]
const max = sorted[sorted.length - 1]
console.log(` avg ${avg.toFixed(0)} ms median ${med} ms max ${max} ms`)
console.log(` (subtract 5x concurrent chattr baseline above to isolate fs.watch event latency + readdir)`)
}
console.log('')
// -------------------------------------------------------------------------
// Phase B — write all VM backups and measure live watcher locking speed.
// 5 probe VMs are tracked individually for per-backup latency under load.
// -------------------------------------------------------------------------
console.log(`=== Phase B: live watcher — writing ${VM_COUNT} backups (${BACKUP_DATE}) ===`)
const probeWriteNotifiers = new Map<string, (tWritten: number) => void>()
const probeBTasks = PROBE_VM_INDICES.map(vmIdx => {
const vm = vms[vmIdx]
const jsonPath = path.join(root, 'xo-vm-backups', vm.uuid, `${BACKUP_DATE}.json`)
return new Promise<{ vmIdx: number; tWritten: number; tLocked: number | null }>(resolve => {
probeWriteNotifiers.set(vm.uuid, tWritten => {
waitUntilImmutable(jsonPath).then(tLocked => resolve({ vmIdx, tWritten, tLocked }))
})
})
})
const tWrite = Date.now()
let written = 0
await asyncEach(
vms,
async vm => {
const vmDir = path.join(root, 'xo-vm-backups', vm.uuid)
await Promise.all(
vm.vdiUuids.map(vdiUuid =>
fs.writeFile(path.join(vmDir, 'vdis', JOB_UUID, vdiUuid, `${BACKUP_DATE}.vhd`), 'fake vhd data')
)
)
await fs.writeFile(path.join(vmDir, `${BACKUP_DATE}.json`), '{}')
written++
probeWriteNotifiers.get(vm.uuid)?.(Date.now())
if (written % 1000 === 0) {
sampleMemory()
console.log(
` ${written}/${VM_COUNT} backups written` +
` | ${Date.now() - tWrite} ms elapsed` +
` | peak RSS ${mb(peakRss)}`
)
}
},
{ concurrency: WRITE_CONCURRENCY }
)
const tWriteDone = Date.now() - tWrite
console.log(`All ${VM_COUNT} backups written in ${tWriteDone} ms`)
console.log('\n Probe VM json→immutable latency (under full load):')
const [probeBResults, phaseB] = await Promise.all([
Promise.all(probeBTasks),
waitForImmutableCount(root, BACKUP_DATE, vmUuids, VM_COUNT, 'live-lock'),
])
for (const { vmIdx, tWritten, tLocked } of probeBResults) {
const latency = tLocked === null ? 'TIMEOUT' : `${tLocked - tWritten} ms`
console.log(` VM[${String(vmIdx).padStart(4)}]: json→immutable ${latency}`)
}
console.log(`Phase B ${phaseB.reached ? 'DONE' : 'TIMEOUT'} in ${phaseB.elapsed} ms`)
console.log('')
// -------------------------------------------------------------------------
// Phase C — lift all immutability
// -------------------------------------------------------------------------
await close()
console.log(`=== Phase C: lift all ${EXPECTED_TOTAL} protected files ===`)
const tLift = Date.now()
let lastLiftLog = Date.now()
const liftSampler = setInterval(() => {
sampleMemory()
const now = Date.now()
if (now - lastLiftLog >= 5000) {
console.log(` lifting in progress... ${Date.now() - tLift} ms elapsed | peak RSS ${mb(peakRss)}`)
lastLiftLog = now
}
}, 1000)
await liftRemoteImmutability(root, 0)
clearInterval(liftSampler)
sampleMemory()
const tLiftDone = Date.now() - tLift
console.log(`Phase C DONE in ${tLiftDone} ms`)
console.log('')
// -------------------------------------------------------------------------
// Results
// -------------------------------------------------------------------------
clearInterval(memSampler)
sampleMemory()
const avgProbeSingle = probeSingleTimes.length > 0 ? `${average(probeSingleTimes).toFixed(0)} ms` : 'N/A'
const avgProbeBLatencies = probeBResults
.map(r => (r.tLocked === null ? null : r.tLocked - r.tWritten))
.filter((v): v is number => v !== null)
const avgProbeB = avgProbeBLatencies.length > 0 ? `${average(avgProbeBLatencies).toFixed(0)} ms` : 'N/A'
console.log('=== Results ===')
console.log(`watchRemote startup: ${tWatcherReady} ms`)
console.log(`Phase A — single probe avg: ${avgProbeSingle} (no contention)`)
console.log(`Phase B — write backups: ${tWriteDone} ms`)
console.log(`Phase B — lock all: ${phaseB.elapsed} ms ${phaseB.reached ? 'OK' : 'TIMEOUT'}`)
console.log(`Phase B — avg lock/backup: ${(phaseB.elapsed / VM_COUNT).toFixed(0)} ms (total / VM_COUNT)`)
console.log(`Phase B — probe avg (5 samples): ${avgProbeB} (under full load)`)
console.log(`Phase C — lift all: ${tLiftDone} ms`)
console.log(`Memory before watch: ${mb(rssBeforeWatch)}`)
console.log(`Memory after watch: ${mb(rssAfterWatch)}`)
console.log(`Peak RSS: ${mb(peakRss)}`)
console.log(`Current RSS: ${mb(process.memoryUsage().rss)}`)
await cleanupRoot(root)
}
main().catch(err => {
console.error(err)
process.exit(1)
})

75
@xen-orchestra/immutable-backups/src/remote.mts Normal file
View File

@@ -0,0 +1,75 @@
import fs from 'node:fs/promises'
import * as File from './file.mjs'
import assert from 'node:assert'
import { join } from 'node:path'
import { createLogger } from '@xen-orchestra/log'
import { type RemoteConfig } from './_loadConfig.mjs'
import { watchRemote as startRemoteWatcher } from './_watcher.mjs'
const { debug, info, warn } = createLogger('xen-orchestra:immutable-backups:remote')
// Verify that the remote filesystem supports immutability by creating,
// modifying, locking, and unlocking a temporary test file.
async function test(remotePath: string): Promise<void> {
await fs.readdir(remotePath)
const testPath = join(remotePath, '.test-immut')
// cleanup
try {
await File.liftImmutability(testPath)
await fs.unlink(testPath)
} catch {
// cleanup can fail if it's the first test — not an issue
}
// can create, modify and delete a file
await fs.writeFile(testPath, `test immut ${new Date()}`)
await fs.writeFile(testPath, `test immut change 1 ${new Date()}`)
await fs.unlink(testPath)
// cannot modify or delete an immutable file
await fs.writeFile(testPath, `test immut ${new Date()}`)
await File.makeImmutable(testPath)
await assert.rejects(fs.writeFile(testPath, `test immut change 2 ${new Date()}`), { code: 'EPERM' })
await assert.rejects(fs.unlink(testPath), { code: 'EPERM' })
// can modify and delete a file after lifting immutability
await File.liftImmutability(testPath)
await fs.writeFile(testPath, `test immut change 3 ${new Date()}`)
await fs.unlink(testPath)
}
// Start watching a backup remote for new files and make them immutable as they are written.
export async function watchRemote(
remoteId: string,
{ root, immutabilityDuration, delayBetweenSizeCheck }: RemoteConfig
): Promise<{ close: () => void }> {
debug('got config ', { remoteId, root, immutabilityDuration })
// test if fs supports immutability
await test(root)
// Write the immutability settings file and lock it so it cannot be tampered with.
const settingPath = join(root, 'immutability.json')
// Lift first in case it was left immutable from a previous run.
try {
await fs.access(settingPath)
await File.liftImmutability(settingPath)
} catch (error) {
if (error.code !== 'ENOENT') {
info('error lifting immutability on current settings', { error })
}
}
await fs.writeFile(
settingPath,
JSON.stringify({
since: Date.now(),
immutable: true,
duration: immutabilityDuration,
})
)
await File.makeImmutable(settingPath)
const close = await startRemoteWatcher(root, err => warn('watcher error', { err }), { delayBetweenSizeCheck })
return { close }
}

15
@xen-orchestra/immutable-backups/tsconfig.json Normal file
View File

@@ -0,0 +1,15 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"module": "ESNext",
"moduleResolution": "bundler",
"target": "es2022",
"skipLibCheck": true,
"types": ["node"],
"declaration": true
},
"include": ["./src/**/*"],
"exclude": ["./node_modules", "./dist"]
}

24
@xen-orchestra/log/index.d.ts vendored Normal file
View File

@@ -0,0 +1,24 @@
/**
* A structured log entry's data payload.
* Typically contains contextual key-value pairs (e.g. `{ vmId, path }`).
*/
type LogData = Record<string, unknown>
/**
* A log method accepts either:
* - a message string with optional structured data
* - an Error instance (message is extracted, error is stored in data)
*/
type LogMethod = (message: string | Error | null, data?: LogData | Error | number | string | unknown) => void
export interface Logger {
fatal: LogMethod
error: LogMethod
warn: LogMethod
info: LogMethod
debug: LogMethod
wrap: <T extends (...args: any[]) => any>(message: string, fn: T) => T
}
export function createLogger(namespace: string): Logger
export default createLogger

1
@xen-orchestra/log/package.json
View File

@@ -36,6 +36,7 @@
"postversion": "npm publish",
"test": "node --test"
},
"types": "./index.d.ts",
"exports": {
".": "./index.js",
"./_*": null,

2
@xen-orchestra/rest-api/src/middlewares/generic-error-handler.middleware.mts
View File

@@ -22,7 +22,7 @@ const log = createLogger('xo:rest-api:error-handler')
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export default function genericErrorHandler(error: unknown, req: Request, res: Response, _next: NextFunction) {
if (!(error instanceof Error)) {
log.error(error)
log.error(JSON.stringify(error))
res.status(500).json({ error })
return
}

2
@xen-orchestra/rest-api/src/vms/vm.service.mts
View File

@@ -130,7 +130,7 @@ export class VmService {
nSuspended++
break
default:
log.warn('Invalid VM power_state', vm.id, vm.power_state)
log.warn('Invalid VM power_state', {vmId: vm.id, vmPowerState : vm.power_state})
nUnknown++
break
}

1
CHANGELOG.unreleased.md
View File

@@ -60,6 +60,7 @@
- @xen-orchestra/disk-cli major
- @xen-orchestra/disk-transform patch
- @xen-orchestra/fs minor
- @xen-orchestra/immutable-backups major
- @xen-orchestra/mcp minor
- @xen-orchestra/qcow2 minor
- @xen-orchestra/rest-api minor

18
docs/docs/immutability.md
View File

@@ -50,6 +50,8 @@ npm install -g @xen-orchestra/immutable-backups
Create a configuration file at `/etc/xo-immutable-backups/config.toml`, with the following structure:
```toml
liftEvery = "1h"
[remotes.remote1]
root = "/mnt/ssd/vhdblock/"
immutabilityDuration = "7d"
@@ -57,18 +59,10 @@ immutabilityDuration = "7d"
#### Mandatory Parameters
- **`root`**: Specifies the directory where Xen Orchestra stores backups.
- **`immutabilityDuration`**: Defines how long files remain protected from deletion (e.g., `7d` for 7 days).
#### Optional Parameters
For additional validation, you can enable:
```toml
rebuildIndexOnStart = true
```
This option scans and validates existing files when the service starts. It can be resource-intensive.
- **`liftEvery`**: Define how often the script will check and lift immutabiltiy (e.g., `1h` for every hour).
- Per remote
- **`root`**: Specifies the directory where Xen Orchestra stores backups.
- **`immutabilityDuration`**: Defines how long files remain protected from deletion (e.g., `7d` for 7 days).
### 3. Start the Service

9
packages/vhd-lib/disk-consumer/DiskConsumerVhdDirectory.mjs
View File

@@ -6,12 +6,12 @@
*/
import { BaseVhd, FULL_BLOCK_BITMAP } from './BaseVhd.mjs'
import { dirname } from 'node:path'
import { v4 as uuidv4 } from 'uuid'
import { basename, dirname } from 'node:path'
import { asyncEach } from '@vates/async-each'
import { VhdDirectory, VhdAbstract } from 'vhd-lib'
import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
import { DEFAULT_BLOCK_SIZE } from '../_constants.js'
import assert from 'node:assert'
/**
* @typedef {Object} VhdRemoteTarget
@@ -45,7 +45,10 @@ export class DiskConsumerVhdDirectory extends BaseVhd {
*/
async write(signal) {
const { handler, path, compression, flags, validator, concurrency } = this.#target
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
const SUFFIX = '.alias.vhd'
assert.ok(path.endsWith(SUFFIX), `filename must be an alias , got ${path}`)
const base = basename(path).substring(0, -SUFFIX.length)
const dataPath = `${dirname(path)}/data/${base}.vhd`
const uid = 'to stream ' + Math.random()
let generator
try {