Compare commits

..

7 Commits

Author SHA1 Message Date
unel
abc3018b66 feat(uiObjectLink): replace all uiObjectLink by uiLink 2026-04-27 14:22:45 +02:00
unel
423f00fefa rebase 2026-04-27 14:22:45 +02:00
unel
7456ab9faf feedback 2026-04-27 14:22:02 +02:00
unel
4d30d43a66 add changelog 2026-04-27 14:22:01 +02:00
unel
d1f3efc57f feat(xo6): rebase and resolve comment 2026-04-27 14:22:01 +02:00
unel
1d881fe8cc feat(xo6/uiLink): wrap uiLink on side panel 2026-04-27 14:22:01 +02:00
unel
7a9de0de57 feat(xo6/uiLink): add text ellipsis on uiLink 2026-04-27 14:21:56 +02:00
646 changed files with 7349 additions and 27652 deletions

View File

@@ -281,7 +281,7 @@ module.exports = {
},
},
{
files: ['@xen-orchestra/{web-core,lite,web}/src/route-map.d.ts'],
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
parserOptions: {
sourceType: 'module',
},

View File

@@ -25,7 +25,7 @@ jobs:
- name: Setup Node environment
uses: actions/setup-node@v4
with:
node-version: '>=22.12'
node-version: '24.14'
cache: 'yarn'
- name: Install project dependencies

View File

@@ -1,7 +1,7 @@
{
"name": "@vates/types",
"private": false,
"version": "1.24.0",
"version": "1.23.0",
"main": "./dist/index.mjs",
"exports": {
".": {

View File

@@ -3,5 +3,4 @@ export * from './xen-api.mjs'
export * from './xo.mjs'
export * from './xo-app.mjs'
export * from './lib/xen-orchestra-xapi.mjs'
export * from './lib/xen-orchestra-acl.mjs'
export * from './lib/complex-matcher.mjs'

View File

@@ -1,78 +0,0 @@
import { Branded } from '../common.mjs'
import type { XoGroup, XoUser } from '../xo.mjs'
export type XoAclRole =
| {
id: Branded<'acl-v2-role'>
name: string
description?: string
}
| {
id: Branded<'acl-v2-role'>
name: string
description?: string
isTemplate: true
roleTemplateId: number
}
export type XoAclSupportedActionsByResource = {
[resource: string]: Record<string, unknown>
}
/**
* E.g
* vms {
* shutdown: {
* clean: true,
* hard: true,
* }
* }
*
* `shutdown | shutdown:clean | shutdown:hard`
*/
export type GetKeysRecursively<T, Prefix extends string = ''> = {
[K in keyof T]: T[K] extends object
? K extends string
? `${Prefix}${K}` | GetKeysRecursively<T[K], `${Prefix}${K}:`>
: never
: K extends string
? `${Prefix}${K}`
: never
}[keyof T]
export type XoAclSupportedResource<TActionsByResource extends XoAclSupportedActionsByResource> =
keyof TActionsByResource
export type XoAclSupportedActions<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = (GetKeysRecursively<TActionsByResource[TResource]> & string) | '*'
export type XoAclBasePrivilege = {
id: Branded<'acl-v2-privilege'>
resource: string
action: string
selector?: string
effect: 'allow' | 'deny'
roleId: XoAclRole['id']
}
export type XoAclPrivilege<
TActionsByResource extends XoAclSupportedActionsByResource,
TResource extends XoAclSupportedResource<TActionsByResource>,
> = XoAclBasePrivilege & {
resource: TResource
action: XoAclSupportedActions<TActionsByResource, TResource>
}
export type XoUserRole = {
id: string
roleId: XoAclRole['id']
userId: XoUser['id']
}
export type XoGroupRole = {
id: string
roleId: XoAclRole['id']
groupId: XoGroup['id']
}

View File

@@ -25,7 +25,6 @@ import type {
} from '../common.mjs'
import type { PassThrough, Readable } from 'node:stream'
import type {
XapiXoRecord,
XoGpuGroup,
XoHost,
XoNetwork,
@@ -37,7 +36,6 @@ import type {
XoVm,
XoVmTemplate,
XoVif,
XoVmSnapshot,
} from '../xo.mjs'
export type XcpPatches = {
@@ -74,19 +72,12 @@ export interface Xapi {
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
connectVif(vifId: XoVif['id']): Promise<void>
getField<T extends XenApiRecord, K extends keyof T>(
type: Extract<WrappedXenApiRecord, T>['$type'],
ref: T['$ref'],
field: K
): Promise<T[K]>
getObject: <
XoRecord extends XapiXoRecord,
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
>(
idOrUuidOrRef: XoRecord['id'] | WrappedRecord['$ref'] | WrappedRecord['uuid'],
defaultValue?: WrappedRecord
) => WrappedRecord
connectVif(vifId: XoVif['id']): Promise<void>
createNetwork(
params:
| {
@@ -130,7 +121,6 @@ export interface Xapi {
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
pool_emergencyShutdown(): Promise<void>
resumeVm(id: XoVm['id']): Promise<void>
revertVm(snapshotId: XoVmSnapshot['id']): Promise<void>
unpauseVm(id: XoVm['id']): Promise<void>
cloneVm(
vmId: XoVm['id'],
@@ -348,13 +338,4 @@ export interface Xapi {
): Promise<XenApiRecord>
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
destroySr(id: XoSr['id']): Promise<void>
xostor_delete(ref: XenApiSr['$ref']): Promise<void>
objects: {
indexes: {
type: {
[XenApiRecord in WrappedXenApiRecord as XenApiRecord['$type']]: Record<XenApiRecord['uuid'], XenApiRecord>
}
}
}
}

View File

@@ -491,9 +491,7 @@ export interface XenApiDrTask {
uuid: string
}
type XenApiHostCallMethods = TagCallMethods & {
<T>(method: 'call_plugin',plugin:string, fn: string, args: Record<string, string>): Promise<T>
}
type XenApiHostCallMethods = TagCallMethods & {}
export interface XenApiHost {
$ref: Branded<'host'>
address: string

View File

@@ -44,10 +44,6 @@ import {
XenApiVifWrapped,
XenApiVmWrapped,
XenApiVtpmWrapped,
XoAclBasePrivilege,
XoAclRole,
XoGroupRole,
XoUserRole,
} from './index.mjs'
export type XapiConnection = Xapi & {
@@ -56,6 +52,7 @@ export type XapiConnection = Xapi & {
sessionId: string
_url?: { protocol: string; hostname: string; port?: string }
}
type XapiRecordByXapiXoRecord = {
gpuGroup: XenApiGpuGroupWrapped
host: XenApiHostWrapped
@@ -81,42 +78,7 @@ type XapiRecordByXapiXoRecord = {
'VM-template': XenApiVmWrapped
VTPM: XenApiVtpmWrapped
}
type LicenseProductId =
| 'premium'
| 'xcpng-enterprise'
| 'xcpng-standard'
| 'xo-proxy'
| 'xosan.trial'
| 'xostor'
| 'xostor.trial'
type LicenseProductType = 'xo' | 'xoproxy' | 'xcpng' | 'xosan' | 'xostor'
type License = {
id: string
type: 'license'
created: number
licenseType: 'unit'
productId: LicenseProductId
tags: string[]
expires?: number
boundObjectId?: string
bindDate?: number
buyer?: { token: string; email: string }
history?: { date: number; boundObjectId: string }[]
rebound?: number
productTypes?: LicenseProductType[]
bundleInfo?: { name: string; id: string }
}
export type XoApp = {
hooks: EventEmitter
_redis: {
get(key: string): Promise<string | null>
mSet(args: string[]): Promise<unknown>
sMembers(key: string): Promise<string[]>
sIsMember(key: string, member: string): Promise<boolean>
}
config: {
get<T = string>(path: string): T
getOptional(path: string): Record<string, string> | undefined
@@ -166,18 +128,7 @@ export type XoApp = {
}
// methods ------------
addAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<XoGroupRole>
addAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<XoUserRole>
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
addApiMethod: <A extends unknown[], R>(
name: string,
method: (...args: A) => Promise<R>,
//we use any since it is a legacy call, might be better typed in another pr
info: {
resolve?: any
params?: any
}
) => () => void // eslint-disable-line @typescript-eslint/no-explicit-any
authenticateUser: (
credentials: { token?: string; username?: string; password?: string },
userData?: { ip?: string },
@@ -187,18 +138,6 @@ export type XoApp = {
checkFeatureAuthorization(featureCode: string): Promise<void>
/* connect a server (XCP-ng/XenServer) */
connectXenServer(id: XoServer['id']): Promise<void>
// TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)
createAclV2Privilege(
privilege: Omit<XoAclBasePrivilege, 'id'>,
options?: {
force?: boolean
}
): Promise<XoAclBasePrivilege>
copyAclV2Role(
id: XoAclRole['id'],
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
): Promise<XoAclRole['id']>
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
createAuthenticationToken(opts: {
client?: {
id?: string
@@ -209,21 +148,11 @@ export type XoApp = {
userId: XoUser['id']
}): Promise<XoAuthenticationToken>
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
deleteAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<boolean>
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
deleteGroup(id: XoGroup['id']): Promise<void>
deleteUser(id: XoUser['id']): Promise<void>
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
/* disconnect a server (XCP-ng/XenServer) */
disconnectXenServer(id: XoServer['id']): Promise<void>
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
getAclV2RolePrivileges(roleId: XoAclRole['id']): Promise<XoAclBasePrivilege[]>
getAclV2Role(id: XoAclRole['id']): Promise<XoAclRole>
deleteAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<boolean>
getAclV2Roles(): Promise<XoAclRole[]>
getAclV2UserPrivileges(userId: XoUser['id']): Promise<XoAclBasePrivilege[]>
getAllGroups(): Promise<XoGroup[]>
getAllProxies(): Promise<XoProxy[]>
getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
@@ -317,20 +246,6 @@ export type XoApp = {
preferences?: Record<string, string>
}
): Promise<void>
updateAclV2Privilege(
privilegeId: XoAclBasePrivilege['id'],
privilege: XoAclBasePrivilege
): Promise<XoAclBasePrivilege>
updateAclV2Role(
roleId: XoAclRole['id'],
role: {
name?: XoAclRole['name']
description?: XoAclRole['description'] | null
},
options?: {
force?: boolean
}
): Promise<XoAclRole>
updateGroup(
id: XoGroup['id'],
updates: {
@@ -339,12 +254,4 @@ export type XoApp = {
): void
getAllXapis(): Record<string, XapiConnection>
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
getLicenses(params?: { productType?: LicenseProductType }): Promise<License[]>
bindLicense(params: { licenseId: string; boundObjectId: string }): Promise<License>
unbindLicense(params: {
productId: LicenseProductId
boundObjectId: string
licenseId: string
data?: Record<string, string>
}): Promise<License>
}

View File

@@ -19,7 +19,6 @@ import type {
VM_POWER_STATE,
} from './common.mjs'
import type * as CMType from './lib/complex-matcher.mjs'
import { XoAclPrivilege, XoAclRole, XoAclSupportedActionsByResource } from './lib/xen-orchestra-acl.mjs'
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
type BaseXapiXo = {
@@ -335,7 +334,6 @@ export type XoNetwork = BaseXapiXo & {
defaultIsLocked: boolean
id: Branded<'network'>
insecureNbd?: boolean
isBonded: boolean
name_description: string
name_label: string
nbd?: boolean
@@ -658,15 +656,13 @@ export type XoTask = {
| 'server'
| 'task'
| 'user'
| 'acl-privilege'
| 'acl-role'
params?: Record<string, unknown>
progress?: number
type?: string
userId?: string
[key: string]: unknown | undefined
}
result?: Record<string, unknown>
result: Record<string, unknown>
start: number
status: 'failure' | 'interrupted' | 'pending' | 'success'
tasks?: XoTask[]
@@ -680,7 +676,7 @@ export type XoUser = {
groups: XoGroup['id'][]
id: Branded<'user'>
name?: string
permission: 'none' | 'admin'
permission: string
pw_hash?: string
preferences: Record<string, string>
}
@@ -854,10 +850,7 @@ export type XapiXoRecord =
| XoVtpm
| XoSm
export type NonXapiXoRecord<
TActionsByResource extends XoAclSupportedActionsByResource = never,
TResource extends string = never,
> =
export type NonXapiXoRecord =
| AnyXoBackupArchive
| AnyXoJob
| AnyXoLog
@@ -871,13 +864,8 @@ export type NonXapiXoRecord<
| XoServer
| XoTask
| XoUser
| XoAclRole
| XoAclPrivilege<TActionsByResource, TResource>
export type XoRecord<
TActionsByResource extends XoAclSupportedActionsByResource = never,
TResource extends string = never,
> = XapiXoRecord | NonXapiXoRecord<TActionsByResource, TResource>
export type XoRecord = XapiXoRecord | NonXapiXoRecord
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,16 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
}
})

View File

@@ -1,47 +0,0 @@
{
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"main": "dist/index.mjs",
"name": "@xen-orchestra/acl",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
"version": "1.0.0",
"license": "AGPL-3.0-or-later",
"private": false,
"type": "module",
"engines": {
"node": ">= 20.18"
},
"exports": {
".": {
"types": "./dist/index.d.mts",
"default": "./dist/index.mjs"
}
},
"scripts": {
"build": "tsc && node scripts/generate-types.mjs",
"clean": "rimraf dist/",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test": "npm run build && node --test ./dist/**/*.test.mjs",
"prebuild": "npm run clean",
"prepublishOnly": "npm run build"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"@vates/types": "^1.24.0",
"rimraf": "^6.0.1",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0"
},
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/acl",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"dependencies": {
"complex-matcher": "^1.0.0"
}
}

View File

@@ -1,55 +0,0 @@
#!/usr/bin/env node
// AUTO-GENERATES dist/generated/privilege-types.d.mts
// Run via `npm run build` (after tsc).
import { mkdirSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../dist/actions/index.mjs'
import { getActionStrings } from '../dist/index.mjs'
function toPascalCase(str) {
return str
.split(/[-_]/)
.map(s => s.charAt(0).toUpperCase() + s.slice(1))
.join('')
}
const resources = Object.keys(SUPPORTED_ACTIONS_BY_RESOURCE)
const typeBlocks = resources.map(resource => {
const actionUnion = getActionStrings(resource)
.map(action => `'${action}'`)
.join(' | ')
return `export type ${toPascalCase(resource)}Privilege = {
id: XoAclBasePrivilege['id']
resource: '${resource}'
action: ${actionUnion}
selector?: XoAclBasePrivilege['selector']
effect: XoAclBasePrivilege['effect']
roleId: XoAclBasePrivilege['roleId']
}`
})
const anyUnion = resources.map(resource => ` | ${toPascalCase(resource)}Privilege`).join('\n')
const types = `\
// -----------------------------------------------------------------------
// AUTO-GENERATED — do not edit manually.
// Regenerated on every \`npm run build\`.
// -----------------------------------------------------------------------
import type { XoAclBasePrivilege } from '@vates/types'
${typeBlocks.join('\n\n')}
export type AnyPrivilege =
${anyUnion}
`
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
const outDir = join(root, 'dist/generated')
mkdirSync(outDir, { recursive: true })
writeFileSync(join(outDir, 'privilege-types.d.mts'), types, 'utf-8')

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
action: true,
effect: true,
resource: true,
selector: true,
},
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
description: true,
groups: true,
name: true,
users: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,9 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
users: true,
},
}

View File

@@ -1,13 +0,0 @@
export default {
'allow-vm': true,
disable: true,
enable: true,
evacuate: true,
export: {
logs: true,
},
read: true,
update: {
tags: true,
},
}

View File

@@ -1,80 +0,0 @@
import { default as aclPrivilege } from './acl-privilege.mjs'
import { default as aclRole } from './acl-role.mjs'
import { default as alarm } from './alarm.mjs'
import { default as backupArchive } from './backup-archive.mjs'
import { default as backupJob } from './backup-job.mjs'
import { default as backupLog } from './backup-log.mjs'
import { default as backupRepository } from './backup-repository.mjs'
import { default as gpuGroup } from './gpuGroup.mjs'
import { default as group } from './group.mjs'
import { default as host } from './host.mjs'
import { default as message } from './message.mjs'
import { default as network } from './network.mjs'
import { default as pbd } from './pbd.mjs'
import { default as pci } from './pci.mjs'
import { default as pgpu } from './pgpu.mjs'
import { default as pif } from './pif.mjs'
import { default as pool } from './pool.mjs'
import { default as proxy } from './proxy.mjs'
import { default as restoreLog } from './restore-log.mjs'
import { default as schedule } from './schedule.mjs'
import { default as server } from './server.mjs'
import { default as sm } from './sm.mjs'
import { default as sr } from './sr.mjs'
import { default as task } from './task.mjs'
import { default as user } from './user.mjs'
import { default as vbd } from './vbd.mjs'
import { default as vdiSnapshot } from './vdi-snapshot.mjs'
import { default as vdiUnmanaged } from './vdi-unmanaged.mjs'
import { default as vdi } from './vdi.mjs'
import { default as vgpu } from './vgpu.mjs'
import { default as vgpuType } from './vgpuType.mjs'
import { default as vif } from './vif.mjs'
import { default as vmController } from './vm-controller.mjs'
import { default as vmSnapshot } from './vm-snapshot.mjs'
import { default as vmTemplate } from './vm-template.mjs'
import { default as vm } from './vm.mjs'
import { default as vtpm } from './vtpm.mjs'
/**
* value without sub action like 'shutdown' mean: 'shutdown:*' (so shutdown mean you can shutdown:clean and shutdown:hard)
*/
export const SUPPORTED_ACTIONS_BY_RESOURCE = {
'acl-privilege': aclPrivilege,
'acl-role': aclRole,
alarm,
'backup-archive': backupArchive,
'backup-job': backupJob,
'backup-log': backupLog,
'backup-repository': backupRepository,
group,
gpuGroup,
host,
message,
network,
pbd,
pci,
pgpu,
pif,
pool,
proxy,
'restore-log': restoreLog,
schedule,
server,
sm,
sr,
task,
user,
vbd,
'vdi-snapshot': vdiSnapshot,
'vdi-unmanaged': vdiUnmanaged,
vdi,
vgpu,
vgpuType,
vif,
'vm-controller': vmController,
'vm-snapshot': vmSnapshot,
'vm-template': vmTemplate,
vm,
vtpm,
} as const

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,8 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
management: true,
},
}

View File

@@ -1,13 +0,0 @@
export default {
create: {
network: true,
vm: true,
},
'emergency-shutdown': true,
read: true,
'rolling-reboot': true,
'rolling-update': true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,4 +0,0 @@
export default {
read: true,
run: true,
}

View File

@@ -1,7 +0,0 @@
export default {
connect: true,
create: true,
delete: true,
disconnect: true,
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,11 +0,0 @@
export default {
delete: true,
import: {
vdi: true,
vm: true,
},
read: true,
update: {
tags: true,
},
}

View File

@@ -1,5 +0,0 @@
export default {
abort: true,
delete: true,
read: true,
}

View File

@@ -1,11 +0,0 @@
export default {
create: true,
delete: true,
read: true,
update: {
name: true,
password: true,
permission: true,
preferences: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,11 +0,0 @@
export default {
boot: true,
create: true,
delete: true,
'export-content': true,
'import-content': true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,4 +0,0 @@
export default {
read: true,
create: true,
}

View File

@@ -1,6 +0,0 @@
export default {
read: true,
update: {
tags: true,
},
}

View File

@@ -1,8 +0,0 @@
export default {
delete: true,
export: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,9 +0,0 @@
export default {
delete: true,
export: true,
instantiate: true,
read: true,
update: {
tags: true,
},
}

View File

@@ -1,24 +0,0 @@
export default {
delete: true,
export: true,
pause: true,
read: true,
reboot: {
clean: true,
hard: true,
},
resume: true,
'revert-snapshot': true,
shutdown: {
clean: true,
hard: true,
},
snapshot: true,
start: true,
suspend: true,
unpause: true,
update: {
datasources: true,
tags: true,
},
}

View File

@@ -1,3 +0,0 @@
export default {
read: true,
}

View File

@@ -1,119 +0,0 @@
import * as CM from 'complex-matcher'
import type { XoAclPrivilege, XoAclSupportedActions, XoAclSupportedResource } from '@vates/types/lib/xen-orchestra/acl'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../actions/index.mjs'
export type SupportedActionsByResource = typeof SUPPORTED_ACTIONS_BY_RESOURCE
export type SupportedResource = XoAclSupportedResource<SupportedActionsByResource>
export type SupportedActions<T extends SupportedResource> = XoAclSupportedActions<SupportedActionsByResource, T>
export type TPrivilege<T extends SupportedResource> = XoAclPrivilege<SupportedActionsByResource, T>
export class Privilege<T extends SupportedResource> {
#action: TPrivilege<T>['action']
#selector?: (object: object) => boolean
#resource: TPrivilege<T>['resource']
#effect: TPrivilege<T>['effect']
constructor({
action,
selector,
resource,
effect,
}: {
action: TPrivilege<T>['action']
selector?: TPrivilege<T>['selector']
resource: TPrivilege<T>['resource']
effect: TPrivilege<T>['effect']
}) {
Privilege.checkActionIsValid(resource, action)
this.#action = action
this.#selector = selector !== undefined ? CM.parse(selector).createPredicate() : undefined
this.#resource = resource
this.#effect = effect
}
get effect() {
return this.#effect
}
#matchAction<Resource extends SupportedResource = T>(action: SupportedActions<Resource>) {
// read:name_label - read:name_label
if (this.#action === action) {
return true
}
const [namespaceToMatch, actionToMatch] = action.split(':')
const [thisNamespace, thisAction] = this.#action.split(':')
if (thisNamespace === '*') {
return true
}
// update:vm - read:name_label
if (thisNamespace !== namespaceToMatch) {
return false
}
// read - read:name_label
if (thisAction === undefined) {
return true
}
// read:name_decription - read:name_label
if (thisAction !== actionToMatch) {
return false
}
throw new Error(`Unable to verify if ${this.#action} match ${action} `)
}
#matchSelector(object: object) {
if (this.#selector === undefined) {
return true
}
return this.#selector(object)
}
#matchResource(resource: string): resource is SupportedResource {
return resource === this.#resource
}
match<Resource extends SupportedResource = T>(constraint: {
action: SupportedActions<Resource>
resource: Resource
object: object
}): boolean {
return (
this.#matchResource(constraint.resource) &&
this.#matchAction(constraint.action) &&
this.#matchSelector(constraint.object)
)
}
/**
* Throw if action not supported
*/
static checkActionIsValid<T extends SupportedResource>(resource: T, action: SupportedActions<T>) {
const supportedActions = SUPPORTED_ACTIONS_BY_RESOURCE[resource]
if (supportedActions === undefined) {
throw new Error(`${resource} resource not supported`)
}
if (action === '*') {
return
}
const segments = action.split(':')
let _action: object | undefined = undefined
segments.forEach(segment => {
_action = (_action ?? supportedActions)[segment]
if (_action === undefined) {
throw new Error(
`${action} action not supported for the resource: ${resource}. See ${JSON.stringify(supportedActions)}`
)
}
})
}
}

View File

@@ -1,10 +0,0 @@
// DO NOT EDIT
// Overwritten by scripts/generate-types.mjs after running npm run build
// scripts/generate-types.mjs will replace this file to automatically generate and export all privileged types.
// This makes external use easier because it's not necessary to resolve the type every time.
import type { Privilege, SupportedResource } from '../index.mjs'
export type AnyPrivilege = {
[Resource in SupportedResource]: Privilege<Resource>
}[SupportedResource]

View File

@@ -1,167 +0,0 @@
import assert from 'node:assert'
import type { XoUser } from '@vates/types/xo'
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
import {
Privilege as CPrivilege,
TPrivilege as Privilege,
SupportedActions,
SupportedActionsByResource,
SupportedResource,
} from './class/privilege.mjs'
// Export all types constructed using SUPPORTED_ACTIONS_BY_RESOURCE
export type { Privilege, SupportedActions, SupportedActionsByResource, SupportedResource }
export { SUPPORTED_ACTIONS_BY_RESOURCE }
export type * from './generated/privilege-types.mjs'
import { AnyPrivilege } from './generated/privilege-types.mjs'
export type AnyPrivilegeOnParam = {
[Resource in SupportedResource]: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: object | object[]
}
}[SupportedResource]
export function hasPrivilegeOn<T extends SupportedResource>({
user,
action,
resource,
objects,
userPrivileges,
}: {
user: XoUser
resource: T
action: SupportedActions<T>
objects: object | object[]
userPrivileges: AnyPrivilege[]
}) {
// Function that will be called outside of the module
// We cannot be sure types are respected
assert.strictEqual(typeof user?.permission, 'string')
assert.strictEqual(typeof action, 'string')
assert.strictEqual(typeof resource, 'string')
assert.strictEqual(objects === undefined, false)
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
if (user.permission === 'admin') {
return true
}
const effectsByObject = new Map<object, Privilege<T>['effect'][]>()
if (Array.isArray(objects)) {
objects.forEach(obj => effectsByObject.set(obj, []))
} else {
effectsByObject.set(objects, [])
}
for (const userPrivilege of userPrivileges) {
const privilege = new CPrivilege(userPrivilege as Privilege<typeof userPrivilege.resource>)
for (const [object, effects] of effectsByObject.entries()) {
if (!privilege.match({ action, resource, object })) {
continue
}
effects.push(privilege.effect)
}
}
if (
Array.from(effectsByObject.values()).some(
effects => effects.length === 0 || effects.some(effect => effect === 'deny')
)
) {
return false
}
return true
}
export function getMissingPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return params
.filter(
param =>
!hasPrivilegeOn({
user: param.user,
resource: param.resource,
action: param.action as SupportedActions<typeof param.resource>,
objects: param.objects,
userPrivileges,
})
)
.map(({ action, objects, resource }) => {
let objectIds: unknown[] | undefined
let objectId: unknown | undefined
if (Array.isArray(objects)) {
if (objects.length > 1) {
objectIds = []
objects.forEach(obj => {
if ('id' in obj) {
objectIds!.push(obj.id)
}
})
} else {
objects = objects?.[0] ?? {}
}
}
if (!Array.isArray(objects)) {
objectId = 'id' in objects ? objects.id : undefined
}
return {
objectId,
objectIds,
action,
resource,
}
})
}
export function hasPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
return getMissingPrivileges(params, userPrivileges).length === 0
}
export function filterObjectsWithPrivilege<Resource extends SupportedResource, Object extends object>(param: {
user: XoUser
resource: Resource
action: SupportedActions<Resource>
objects: Object[]
userPrivileges: AnyPrivilege[]
}) {
return param.objects.filter(obj => hasPrivilegeOn({ ...param, objects: obj }))
}
// Utils
/**
* Mirror GetKeysRecursively from @vates/types
*/
export function getActionStrings<Resource extends SupportedResource, Actions = SupportedActions<Resource>>(
resource: Resource
): Actions[] {
type NestedActions = { [key: string]: boolean | NestedActions }
function collectActions(obj: NestedActions, prefix?: Actions): Actions[] {
const actions: Actions[] = ['*' as Actions]
for (const [key, value] of Object.entries(obj)) {
const action = (prefix !== undefined ? `${prefix}:${key}` : key) as Actions
actions.push(action)
if (typeof value === 'boolean') {
continue
}
actions.push(...collectActions(value, action))
}
return actions
}
return collectActions(SUPPORTED_ACTIONS_BY_RESOURCE[resource])
}

View File

@@ -1,378 +0,0 @@
import { test, suite } from 'node:test'
import assert from 'node:assert'
import type { Branded } from '@vates/types/common'
import type { XoUser } from '@vates/types/xo'
import { Privilege } from '../class/privilege.mjs'
import type { Privilege as TPrivilege } from '../index.mjs'
import {
filterObjectsWithPrivilege,
getActionStrings,
getMissingPrivileges,
hasPrivilegeOn,
hasPrivileges,
} from '../index.mjs'
suite('Privilege.checkActionIsValid behavior', () => {
test('throw on invalid resource', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('foo', 'read'))
})
test('throw on invalid action', () => {
// @ts-expect-error test invalid action
assert.throws(() => Privilege.checkActionIsValid('vm', 'foo:vm'))
})
test('should work', () => {
assert.doesNotThrow(() => Privilege.checkActionIsValid('vm', 'read'))
})
})
suite('Privilege.match behavior', () => {
const privilege = new Privilege({ action: 'read', resource: 'vm', effect: 'allow' })
const object = { id: 1, name_label: 'foo' }
test('Should not match because of the action', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'start', object }), false)
})
test('Should not match because of the resource', () => {
assert.strictEqual(privilege.match({ resource: 'host', action: 'read', object }), false)
})
test('Should match', () => {
assert.strictEqual(privilege.match({ resource: 'vm', action: 'read', object }), true)
})
test('Should match even if the action is not equal to the action privilege', () => {
const shutdownPrivilege = new Privilege({ action: 'shutdown', resource: 'vm', effect: 'allow' })
assert.strictEqual(shutdownPrivilege.match({ resource: 'vm', action: 'shutdown:clean', object }), true)
})
test('Should match because the action is *', () => {
const vmPrivilege = new Privilege({ action: '*', resource: 'vm', effect: 'allow' })
assert.strictEqual(vmPrivilege.match({ action: 'reboot:hard', resource: 'vm', object: object }), true)
})
test('Should not match due to insufficient privilege', () => {
const cleanShutdownPrivilege = new Privilege({ action: 'shutdown:clean', resource: 'vm', effect: 'allow' })
assert.strictEqual(cleanShutdownPrivilege.match({ resource: 'vm', action: 'shutdown', object }), false)
})
})
suite('ACL V2 behavior', async () => {
// === Setup ===
const admin = {
id: '1' as XoUser['id'],
groups: [],
permission: 'admin',
email: 'admin@admin.net',
preferences: {},
} satisfies XoUser
const user = {
id: '2' as XoUser['id'],
groups: [],
permission: 'none',
email: 'user@user.net',
preferences: {},
} satisfies XoUser
const privilegeId = 'fake-priv-test-id' as Branded<'acl-v2-privilege'>
const roleId = 'fake-role-test-id' as Branded<'acl-v2-role'>
// === Objects
const poolWithoutPrivilege = { id: 'pool-id' }
const alpineVm = { id: 'alpine', creation: { creator: admin.id } }
const almaVm = { id: 'alma', creation: { creator: user.id } }
const xoaVm = { id: 'xoa' }
const template = { id: 'template-id', $pool: '1' }
const templateDeny = { id: 'template-deny-id', $pool: '2' }
const vdis = [{ $SR: '2' }, { $SR: '2' }]
const vdisMultipleSrs = [{ $SR: '1' }, { $SR: '2' }]
const vifs = [{ $network: '1' }]
const iso = { id: 'iso-vdi-id', $SR: '3' }
const hostAffinity = { id: 'affinity-host-id', $pool: '1' }
// === DB Privileges
const vmRead = { id: privilegeId, action: 'read', resource: 'vm', effect: 'allow', roleId } satisfies TPrivilege<'vm'>
const vmCleanShutdown = {
id: privilegeId,
action: 'shutdown:clean',
resource: 'vm',
selector: `creation:creator:${user.id}`,
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm'>
const xoaVmDenyRead = {
id: privilegeId,
action: 'read',
resource: 'vm',
effect: 'deny',
selector: `id: ${xoaVm.id}`,
roleId,
} satisfies TPrivilege<'vm'>
const vdiCreate = {
id: privilegeId,
action: 'create',
resource: 'vdi',
selector: '$SR:2',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const vifCreate = {
id: privilegeId,
action: 'create',
resource: 'vif',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vif'>
const templateInstantiate = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vm-template'>
const templateInstantiateDeny = {
id: privilegeId,
action: 'instantiate',
resource: 'vm-template',
selector: '$pool:2',
effect: 'deny',
roleId,
} satisfies TPrivilege<'vm-template'>
const isoVdiUse = {
id: privilegeId,
action: 'boot',
resource: 'vdi',
selector: '$SR:3',
effect: 'allow',
roleId,
} satisfies TPrivilege<'vdi'>
const hostVm = {
id: privilegeId,
action: 'allow-vm',
resource: 'host',
effect: 'allow',
roleId,
} satisfies TPrivilege<'host'>
const allPrivileges = [
vmRead,
vmCleanShutdown,
xoaVmDenyRead,
vdiCreate,
vifCreate,
templateInstantiate,
templateInstantiateDeny,
isoVdiUse,
hostVm,
]
// === Test ===
suite('hasPrivilegeOn behavior', () => {
test('Should always return true for admin', () => {
assert.strictEqual(
hasPrivilegeOn<'pool'>({
user: admin,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
true
)
})
test('Should return false by default for non admin', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'pool',
objects: poolWithoutPrivilege,
userPrivileges: allPrivileges,
}),
false
)
})
test('Should return true if user has privilege', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: alpineVm, userPrivileges: allPrivileges }),
true
)
})
test('Should return false if user have a both deny and allow privileges', () => {
assert.strictEqual(
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: xoaVm, userPrivileges: allPrivileges }),
false
)
})
test('Should handle multiple and single object', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm],
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'read',
resource: 'vm',
objects: [alpineVm, almaVm, xoaVm],
userPrivileges: allPrivileges,
}),
false
)
})
test('Should allow clean shutdown only own VMs', () => {
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: almaVm,
userPrivileges: allPrivileges,
}),
true
)
assert.strictEqual(
hasPrivilegeOn({
user,
action: 'shutdown:clean',
resource: 'vm',
objects: [almaVm, alpineVm],
userPrivileges: allPrivileges,
}),
false
)
})
})
suite('getMissingPrivileges behavior', () => {
test('Should return empty array if no missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'shutdown:clean', resource: 'vm', objects: almaVm },
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 0)
})
test('Should return only missing privileges', () => {
const missingPrivileges = getMissingPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege },
{ user, action: '*', resource: 'pool', objects: poolWithoutPrivilege },
],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 2)
})
test('Should return objectId if only one object in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectId, undefined)
})
test('Should return objectIds if multiple objects in the missing privilege', () => {
const missingPrivileges = getMissingPrivileges(
[{ user, action: 'read', resource: 'pool', objects: [poolWithoutPrivilege, poolWithoutPrivilege] }],
allPrivileges
)
assert.strictEqual(missingPrivileges.length, 1)
assert.notStrictEqual(missingPrivileges[0]!.objectIds, undefined)
})
})
suite('hasPrivileges behavior', () => {
test('Should return false if at least one privilege deny', () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: templateDeny },
{ user, action: 'create', resource: 'vdi', objects: vdis },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
test("Should return false if at least one privilege didn't match", () => {
const result = hasPrivileges(
[
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
{ user, action: 'create', resource: 'vdi', objects: vdisMultipleSrs },
{ user, action: 'create', resource: 'vif', objects: vifs },
{ user, action: 'boot', resource: 'vdi', objects: iso },
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
],
allPrivileges
)
assert.strictEqual(result, false)
})
})
suite('filterObjectsWithPrivilege behavior', () => {
test('Should return only objects that match privileges', () => {
let vms = [almaVm, alpineVm, xoaVm, xoaVm]
assert.strictEqual(vms.length, 4)
vms = filterObjectsWithPrivilege({
user,
action: 'read',
resource: 'vm',
objects: vms,
userPrivileges: allPrivileges,
})
assert.strictEqual(vms.length, 2)
})
})
})
suite('getActionStrings', () => {
test('returns flat actions for a resource without nesting', () => {
const actions = getActionStrings('host')
assert.ok(actions.includes('read'))
// @ts-ignore A TypeScript error is thrown because `foo` is not of type `SupportedActions<'vm'>`.
// To test for a negative value, the error is ignored.
assert.strictEqual(actions.includes('foo'), false)
})
test('parent action is included alongside its children', () => {
const actions = getActionStrings('vm')
assert.ok(actions.includes('shutdown'))
assert.ok(actions.includes('shutdown:clean'))
assert.ok(actions.includes('shutdown:hard'))
})
})

View File

@@ -1,19 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"module": "nodenext",
"lib": ["ESNext"],
"moduleResolution": "nodenext",
"target": "ES2022",
"skipLibCheck": true,
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
"noImplicitAny": false, // otherwise it will throw an error when importing packages without type definition
"noUncheckedIndexedAccess": true,
"declaration": true
},
"include": ["./src/**/*"],
"exclude": ["./node_modules", "./dist"]
}

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,21 +0,0 @@
// @ts-check
import eslint from '@eslint/js'
import tseslint from 'typescript-eslint'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
languageOptions: {
parserOptions: {
tsconfigRootDir: __dirname,
},
},
ignores: ['dist'],
rules: {
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
'@typescript-eslint/no-empty-object-type': 'off',
},
})

View File

@@ -1,55 +0,0 @@
{
"name": "@xen-orchestra/backup-archive",
"version": "1.0.0",
"description": "Helper to handle generic backup archives",
"license": "ISC",
"author": "",
"type": "module",
"main": "./dist/index.mjs",
"exports": {
".": "./dist/index.mjs",
"./disks": "./dist/disks/index.mjs"
},
"devDependencies": {
"@eslint/js": "^9.19.0",
"rimraf": "^6.0.1",
"@types/node": "^22.3.0",
"typescript": "~5.6",
"typescript-eslint": "^8.23.0",
"fs-extra": "^11.1.0",
"sinon": "^18.0.0",
"tar-stream": "^3.1.8",
"@types/tar-stream": "^3.1.4",
"tmp": "^0.2.1",
"@vates/types": "^1.21.0"
},
"dependencies": {
"@xen-orchestra/log": "^0.7.1",
"@xen-orchestra/fs": "^4.6.7",
"@xen-orchestra/disk-transform": "^1.2.3",
"@vates/task": "0.7.0",
"golike-defer": "0.5.1",
"uuid": "^9.0.0",
"promise-toolbox": "^0.21.0",
"@vates/async-each": "^1.0.1",
"vhd-lib": "^4.14.7"
},
"scripts": {
"build": "yarn run clean && tsc",
"dev": "tsc --watch",
"postversion": "npm publish --access public",
"test-integration": "yarn run clean && tsc && node --test **/*.integ.mjs",
"clean": "rimraf dist/"
},
"private": true,
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@xen-orchestra/backup-archive",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=20.17"
}
}

View File

@@ -1,409 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import {
MergeRemoteDisk,
openDisk,
openDisposableDisk,
RemoteDisk,
isDisk,
openDiskChain,
} from '@xen-orchestra/backup-archive/disks'
import Disposable from 'promise-toolbox/Disposable'
import {
CleanResult,
DEFAULT_MERGE_CONCURRENCY,
DEFAULT_REMOVE_CONCURRENCY,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { asyncEach } from '@vates/async-each'
import { Task } from '@vates/task'
import { limitConcurrency } from 'limit-concurrency-decorator'
import assert from 'node:assert/strict'
const defaultMergeLimiter = limitConcurrency(1)
/**
* Tracks the disk chain for a single VDI across all backup snapshots.
* Owns merge and deletion decisions for its chain given which disks are still
* referenced by active backups (accumulated via addActiveDiskPath).
*/
export class RemoteDiskLineage {
#handler: RemoteHandlerAbstract
#vdiDir: string
#opts: ResolvedBackupCleanOptions
// Concrete disk paths discovered in this VDI directory (normalized)
#diskPaths: Set<string> = new Set()
// Disk paths that exist on disk but failed to open (corrupted)
#brokenDiskPaths: Set<string> = new Set()
// child path: parent path (from disk header)
#parentOf: Map<string, string> = new Map()
// parent path: child path
#childOf: Map<string, string> = new Map()
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
#activeDiskPaths: Set<string> = new Set()
// Interrupted merges: normalized parent path { stateFilePath, chain }
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
this.#handler = handler
this.#vdiDir = vdiDir
this.#opts = opts
}
#unregisterDisk(diskPath: string): void {
this.#diskPaths.delete(diskPath)
const parent = this.#parentOf.get(diskPath)
if (parent !== undefined) {
this.#parentOf.delete(diskPath)
this.#childOf.delete(parent)
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
this.#childOf.delete(diskPath)
this.#parentOf.delete(child)
}
}
/**
* Lists disk files and reads their headers to build the parent-child chain.
* Also detects interrupted merge state files.
*/
async init(): Promise<void> {
const files = await this.#handler.list(this.#vdiDir, { prependDir: true })
for (const filePath of files) {
if (isDisk(filePath)) {
this.#diskPaths.add(normalize(filePath))
}
}
const uuidToPath = new Map<string, string>()
for (const diskPath of this.#diskPaths) {
let disk: RemoteDisk | undefined
try {
disk = await openDisk({ handler: this.#handler, path: diskPath })
} catch (error) {
if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
throw error
}
this.#opts.logWarn('failed to open disk', { path: diskPath, error })
// Broken disks will be removed by the clean phase
// because they are marked as NOT active
this.#brokenDiskPaths.add(diskPath)
continue
}
try {
const uuid = disk.getUuid()
// Detect Disks with the same UUIDs
// Due to a bug introduced in a1bcd35e2
const existingPath = uuidToPath.get(uuid)
if (existingPath !== undefined) {
this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
let existingDisk: RemoteDisk | undefined
try {
existingDisk = await openDisk({ handler: this.#handler, path: existingPath })
if (existingDisk.containsAllDataOf(disk)) {
this.#opts.logWarn('should drop duplicate disk, existing is superset', { dropped: diskPath })
continue
} else if (disk.containsAllDataOf(existingDisk)) {
this.#opts.logWarn('should drop old duplicate disk, new is superset', { dropped: existingPath })
} else {
this.#opts.logWarn('duplicate disks have different content, should first-seen', { dropped: diskPath })
continue
}
} catch (error) {
this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
this.#unregisterDisk(existingPath)
} finally {
await existingDisk?.close()
}
}
uuidToPath.set(uuid, diskPath)
if (disk.isDifferencing()) {
const parentPath = disk.getParentPath()
this.#parentOf.set(diskPath, parentPath)
if (this.#childOf.has(parentPath)) {
const error = new Error('this script does not support multiple children', {
cause: {
parent: parentPath,
child1: this.#childOf.get(parentPath),
child2: diskPath,
},
})
throw error
}
this.#childOf.set(parentPath, diskPath)
}
} catch (error) {
this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
} finally {
await disk.close()
}
}
this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
this.#handler,
this.#vdiDir,
files,
uuidToPath,
this.#childOf
)
const propagateBroken = (diskPath: string) => {
if (!this.#brokenDiskPaths.has(diskPath)) {
this.#brokenDiskPaths.add(diskPath)
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
propagateBroken(child)
}
}
}
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent) || this.#brokenDiskPaths.has(parent)) {
this.#opts.logWarn('disk broken: parent missing or broken', { child, parent })
propagateBroken(child)
}
}
}
/**
* Returns true if the disk path exists on disk and opened without errors during init.
* Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
*/
isPathAccessible(diskPath: string): boolean {
return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
}
/** Returns true if the disk exists but failed to open during init. */
isBroken(diskPath: string): boolean {
return this.#brokenDiskPaths.has(diskPath)
}
/** Verifies that every parent referenced in disk headers exists in the directory. */
async check(): Promise<void> {
for (const [child, parent] of this.#parentOf) {
if (!this.#diskPaths.has(parent)) {
this.#opts.logWarn('parent disk is missing', { child, parent })
}
}
}
/**
* Declares disk paths as active for this lineage.
* Called by each archive that references this lineage after it determines it is complete.
* Accumulated across all referencing archives before clean() is called.
*/
addActiveDiskPath(diskPath: string): void {
this.#activeDiskPaths.add(normalize(diskPath))
}
/**
* For each orphan disk:
* - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
* - If it has no active descendant → delete it (if opts.remove).
*
* Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
* Deletes orphan merge state files (for missing disks) when opts.remove is true.
*
* Requires activeDiskPaths to be completed to have been called by all referencing archives before this.
* @returns Set of deleted disk paths.
*/
async clean({
remove = this.#opts.remove ?? false,
merge = this.#opts.merge ?? false,
}: { remove?: boolean; merge?: boolean } = {}): Promise<CleanResult> {
const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
const toDelete = new Set<string>()
const toMerge: { chain: string[]; isResuming: boolean }[] = []
const visited = new Set<string>()
// Walk from an orphan toward its descendants.
// Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
// or undefined if the subtree is entirely orphaned (mark for deletion).
const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
if (visited.has(diskPath)) {
return undefined
}
visited.add(diskPath)
if (!orphanDisks.has(diskPath)) {
// Active disk, anchor of the merge chain
return [diskPath]
}
const child = this.#childOf.get(diskPath)
if (child !== undefined) {
const chain = getUsedChildChainOrDelete(child)
if (chain !== undefined) {
chain.unshift(diskPath)
return chain
}
}
toDelete.add(diskPath)
return undefined
}
// Process interrupted merges first so their disks are protected from the orphan loop
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
if (!this.#diskPaths.has(parentPath)) {
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
if (remove) {
this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
await this.#handler.unlink(stateFilePath)
}
continue
}
let chain: string[] | undefined
if (stateChain !== undefined) {
const existing = stateChain.filter(p => this.#diskPaths.has(p))
if (existing.length >= 2) chain = existing
}
if (chain !== undefined) {
chain.forEach(p => visited.add(p))
toMerge.push({ chain, isResuming: true })
}
}
for (const orphan of orphanDisks) {
if (!visited.has(orphan)) {
const parent = this.#parentOf.get(orphan)
if (parent === undefined || !orphanDisks.has(parent)) {
const chain = getUsedChildChainOrDelete(orphan)
if (chain !== undefined && chain.length > 1) {
toMerge.push({ chain, isResuming: false })
}
}
}
}
if (remove) {
await asyncEach(
Array.from(toDelete),
async path => {
this.#opts.logInfo('deleting unused disk', { path })
try {
const disk = await openDisk({ handler: this.#handler, path, ignoreBlockIndexes: true })
await disk.unlink({ force: true })
} catch (error) {
this.#opts.logWarn('failed to delete unused disk', { path, error })
}
},
{ concurrency: DEFAULT_REMOVE_CONCURRENCY }
)
}
if (toMerge.length > 0) {
this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
}
// mergeTargetPath: final size of the disk everything was merged into
const mergedSizes = new Map<string, number>()
if (merge) {
const limitedMergeChain = defaultMergeLimiter(this.#mergeChain.bind(this))
const doMerge = async () => {
await asyncEach(
toMerge,
async ({ chain, isResuming }) => {
const { finalDiskSize, mergeTargetPath } = await limitedMergeChain(chain, isResuming)
mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
},
{ concurrency: this.#opts.mergeConcurrency ?? DEFAULT_MERGE_CONCURRENCY }
)
}
if (toMerge.length !== 0) {
await Task.run({ properties: { name: 'merge' } }, doMerge)
}
}
await this.#cleanOrphanDataFiles(remove)
return {
removedFiles: Array.from(toDelete),
merge: toMerge.length > 0,
size: [...mergedSizes.values()].reduce((total, subSize) => total + subSize, 0),
mergedSizes,
}
}
/**
* Merges ancestors into the active child at the end of the chain.
* Returns the final size of the merge target and its path.
*/
async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
assert.ok(chain.length >= 2, `look to merge a chain shorter than 2 ${JSON.stringify(chain)}`)
const parentPath = chain[0]
this.#opts.logInfo('merging disk chain', { chain: [...chain] }) // need a copy here to log full chain
// The last disk in the chain is the active one that everything gets merged into
const mergeTargetPath = chain.pop()!
const merger = new MergeRemoteDisk(this.#handler, {
logInfo: this.#opts.logInfo,
removeUnused: this.#opts.remove,
mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
onProgress: this.#opts.onProgress,
})
let parentDisk: RemoteDisk | undefined
let childDisk: RemoteDisk | undefined
try {
parentDisk = await openDisk({ handler: this.#handler, path: parentPath, force: isResuming })
childDisk = await openDiskChain({
handler: this.#handler,
path: mergeTargetPath,
until: parentPath,
force: isResuming,
})
const { finalDiskSize } = (await merger.merge(parentDisk, childDisk!))!
return { finalDiskSize, mergeTargetPath }
} finally {
await parentDisk?.close().catch(error => {
this.#opts.logWarn('Error closing parent Disk', error)
})
await childDisk?.close().catch(error => {
this.#opts.logWarn('Error closing child Disk', error)
})
}
}
/**
* Validates each disk integrity, then scans discovered data subdirs
* and deletes any files not associated to a known disk.
*/
async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
const claimedFiles = new Set<string>()
for (const diskPath of this.#diskPaths) {
const claimed = await Disposable.use(
openDisposableDisk({ handler: this.#handler, path: diskPath, ignoreBlockIndexes: true }),
async disk => {
await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
return disk.listAssociatedFiles(this.#vdiDir)
}
)
for (const f of claimed) claimedFiles.add(normalize(f))
}
const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
for (const dataDir of dataDirs) {
const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
for (const item of items) {
if (!claimedFiles.has(normalize(item))) {
this.#opts.logWarn('orphaned data file', { path: item })
if (remove) {
await this.#handler.unlink(item).catch(async err => {
if (err?.code === 'EISDIR') {
await this.#handler.rmtree(item)
} else {
this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
}
})
}
}
}
}
}
}

View File

@@ -1,69 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
export interface PartialBackupMetadata {
mode: 'full' | 'delta'
xva?: string
vhds?: Record<string, string>
vdis?: Record<string, unknown>
isVhdDifferencing?: Record<string, boolean>
size?: number
jobId: string
scheduleId: string
timestamp: number
}
export const DEFAULT_MERGE_CONCURRENCY = 1
export const DEFAULT_REMOVE_CONCURRENCY = 4
export interface BackupCleanOptions {
fix?: boolean
merge?: boolean
remove?: boolean
mergeBlockConcurrency?: number
mergeConcurrency?: number
onProgress?: (progress: { total: number; done: number }) => void
logInfo?: (message: any, opts?: object) => void
logWarn?: (message: any, opts?: object) => void
}
export type ResolvedBackupCleanOptions = BackupCleanOptions & {
logInfo: (message: any, opts?: object) => void
logWarn: (message: any, opts?: object) => void
}
export interface ArchiveCleanOptions {
remove?: boolean
merge?: boolean
mergedSizes?: Map<string, number>
}
export interface CheckResult {
isValid: boolean
// Files referenced by metadata but absent on disk
missingDisks?: string[]
// Disk files that exist but failed to open (corrupted)
brokenDisks?: string[]
// Files present on disk but not referenced by metadata
orphans?: string[]
// Files referenced by at least one valid backup
linked?: string[]
}
export type CleanResult = {
removedFiles: string[]
merge?: boolean
size?: number
mergedSizes?: Map<string, number>
}
export interface VmBackupInterface {
handler: RemoteHandlerAbstract
// metadataPath: string
// metadata: PartialBackupMetadata
rootPath: string
opts: BackupCleanOptions
init(): Promise<void>
check(): Promise<CheckResult>
clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
getAssociatedFiles(opts: object): Array<string>
}

View File

@@ -1,284 +0,0 @@
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, normalize } from '@xen-orchestra/fs/path'
import { resolve } from 'node:path'
import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
import {
ArchiveCleanOptions,
BackupCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
DEFAULT_MERGE_CONCURRENCY,
} from './VmBackup.types.mjs'
import { cleanOrphanDiskDirs } from '@xen-orchestra/backup-archive/disks'
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { promisify } from 'node:util'
import zlib from 'node:zlib'
const gzip = promisify(zlib.gzip)
const gunzip = promisify(zlib.gunzip)
const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
export class VmBackupDirectory implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
files: Array<string> = new Array()
orphans: Set<string> = new Set()
backupArchives: Map<string, VmBackupInterface> = new Map()
opts: ResolvedBackupCleanOptions
// Cached result of the last check() call; invalidated by init()
#checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
#uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
constructor(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions = {
fix: true,
merge: false,
remove: false,
logInfo,
logWarn,
}
) {
this.handler = handler
this.rootPath = vmBackupPath
this.opts = {
...opts,
fix: opts.fix ?? true,
merge: opts.merge ?? false,
remove: opts.remove ?? false,
logInfo: opts.logInfo ?? logInfo,
logWarn: opts.logWarn ?? logWarn,
}
}
async #readCache(path: string): Promise<Record<string, unknown> | undefined> {
try {
return JSON.parse((await gunzip(await this.handler.readFile(path))).toString())
} catch (error) {
if (error?.code !== 'ENOENT') {
logWarn('failed to read cache', { error, path })
}
}
}
async #writeCache(path: string, data: Record<string, unknown>): Promise<void> {
try {
await this.handler.writeFile(path, await gzip(JSON.stringify(data)), { flags: 'w' })
} catch (error) {
logWarn('failed to write cache', { error, path })
}
}
async init() {
this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
this.#checkResult = undefined
this.#uniqueLineages = new Map()
for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
let metadata: PartialBackupMetadata | undefined = undefined
try {
metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
} catch (error) {
this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
}
if (metadata !== undefined) {
try {
const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
this.backupArchives.set(fullPath, backupArchive)
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
}
}
}
}
getAssociatedFiles({ prefix = false }) {
const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
return prefix ? files : files.map(file => basename(file))
}
async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
await this.#checkCacheCount()
for (const backupArchive of this.backupArchives.values()) {
await backupArchive.check()
}
// allUsedFiles is used for root-level orphan detection only.
// Active disk paths are accumulated per-lineage by each archive during check() above.
const allUsedFiles = new Set<string>([
...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
...this.getAssociatedFiles({ prefix: true }),
])
for (const lineage of this.#uniqueLineages!.values()) {
await lineage.check()
}
const orphans = this.files.filter(file => !allUsedFiles.has(file))
const linked = Array.from(allUsedFiles)
this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
return this.#checkResult
}
async clean({
remove = this.opts.remove ?? false,
merge = this.opts.merge ?? false,
}: ArchiveCleanOptions = {}): Promise<CleanResult> {
// Use cached check result if available, otherwise run check now
const { orphans } = this.#checkResult ?? (await this.check())
let cacheNeedsRegen = false
let someLineageMergedOrShouldBe = false
// Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
const allMergedSizes = new Map<string, number>()
await asyncEach(
Array.from(this.#uniqueLineages!.entries()),
async ([_vdiDir, lineage]) => {
const { mergedSizes, removedFiles, merge: hasPendingMerge } = await lineage.clean({ remove, merge })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
if (hasPendingMerge) {
someLineageMergedOrShouldBe = true
}
for (const [diskPath, size] of mergedSizes || []) {
allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
}
},
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
)
// Delete VDI directories not referenced by any archive
const coveredDirs = new Set(this.#uniqueLineages!.keys())
await cleanOrphanDiskDirs(this.handler, this.rootPath, coveredDirs, {
remove,
logWarn: this.opts.logWarn,
logInfo: this.opts.logInfo,
})
// Let each archive clean its own files (e.g. remove metadata for incomplete backups)
// and update metadata with merged sizes if applicable
await asyncEach(
Array.from(this.backupArchives.values()),
async (archive: VmBackupInterface) => {
const { removedFiles } = await archive.clean({ remove, mergedSizes: allMergedSizes })
if (removedFiles.length > 0) {
cacheNeedsRegen = true
}
},
{ concurrency: 2 }
)
if (allMergedSizes.size > 0 || cacheNeedsRegen) {
await this.#regenerateCache()
}
// Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
if (remove) {
await asyncEach(
orphans,
async (orphan: string) => {
try {
await this.handler.unlink(orphan)
} catch (error) {
if (error?.code === 'EISDIR') {
this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
} else if (error?.code !== 'ENOENT') {
throw error
}
}
},
{ concurrency: 2 }
)
}
const size = [...allMergedSizes.values()].reduce((total, merged) => total + merged, 0)
return { removedFiles: orphans, merge: someLineageMergedOrShouldBe, size: size }
}
async #checkCacheCount(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const existingCache = await this.#readCache(cachePath)
const actual = existingCache === undefined ? 0 : Object.keys(existingCache).length
const expected = this.backupArchives.size
if (actual !== expected) {
this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
}
}
async #regenerateCache(): Promise<void> {
const cachePath = `${this.rootPath}/cache.json.gz`
const cache: Record<string, object> = {}
for (const [path, archive] of this.backupArchives.entries()) {
const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
cache[path] = { _filename: path, id: path, ...a.metadata }
}
await this.#writeCache(cachePath, cache)
}
async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
let backupArchive: VmBackupInterface
try {
if (metadata.mode === 'full') {
backupArchive = new VmFullBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
resolve('/', this.rootPath, metadata.xva!),
this.opts
)
} else if (metadata.mode === 'delta') {
const rawDiskPaths = metadata.vhds
const diskPaths =
rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
backupArchive = new VmIncrementalBackupArchive(
this.handler,
this.rootPath,
metadataPath,
metadata,
diskPaths,
this.opts,
this.#uniqueLineages
)
} else {
throw new Error(`Mode ${metadata.mode} not supported`)
}
} catch (error) {
this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
}
await backupArchive.init()
return backupArchive
}
/**
* Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
* The `lock` option is accepted but ignored (locking is the caller's responsibility).
*/
static async cleanVm(
handler: RemoteHandlerAbstract,
vmBackupPath: string,
opts: BackupCleanOptions & { lock?: boolean } = {}
) {
const { lock: _lock, ...cleanOpts } = opts
const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
await dir.init()
await dir.check()
const { merge, size } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
return { merge, size }
}
}

View File

@@ -1,147 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
import { rimraf } from 'rimraf'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import tar from 'tar-stream'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, vmBackupDir
const vmUuid = 'test-vm-uuid'
const rootPath = `xo-vm-backups/${vmUuid}`
async function createMinimalXva() {
const pack = tar.pack()
pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
pack.finalize()
const chunks = []
for await (const chunk of pack) {
chunks.push(chunk)
}
return Buffer.concat(chunks)
}
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
await fs.mkdirp(`${tempDir}/${rootPath}`)
vmBackupDir = new VmBackupDirectory(handler, rootPath)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
async function createFullBackupMetadata(name, xvaName) {
await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
const size = await handler.getSize(`${rootPath}/${xvaName}`)
const metadata = {
mode: 'full',
xva: `./${xvaName}`,
vm: { uuid: vmUuid },
jobId: uniqueId(),
timestamp: Date.now(),
size,
}
await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
return metadata
}
describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
test('init() loads full backup archives from metadata files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await createFullBackupMetadata('backup2.json', 'backup2.xva')
await vmBackupDir.init()
assert.equal(vmBackupDir.backupArchives.size, 2)
})
test('getAssociatedFiles() returns cache files', async () => {
// Create a cache file
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
await vmBackupDir.init()
const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
assert.equal(filesWithPrefix.length, 1)
assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
})
test('check() identifies orphan files', async () => {
// Create a valid backup
await createFullBackupMetadata('backup1.json', 'backup1.xva')
// Create orphan files (not referenced by any metadata)
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
await vmBackupDir.init()
const { orphans, linked } = await vmBackupDir.check()
assert.ok(orphans.some(f => f.includes('orphan.xva')))
assert.ok(orphans.some(f => f.includes('random-file.txt')))
assert.ok(linked.some(f => f.includes('backup1.json')))
assert.ok(linked.some(f => f.includes('backup1.xva')))
})
test('clean() removes orphan files', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.json'))
assert.ok(remainingFiles.includes('backup1.xva'))
assert.ok(!remainingFiles.includes('orphan.xva'))
})
test('clean() preserves cache.json.gz', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
await vmBackupDir.init()
await vmBackupDir.clean()
const remainingFiles = await handler.list(rootPath)
assert.equal(remainingFiles.length, 3)
assert.ok(remainingFiles.includes('cache.json.gz'))
})
test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
})
test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
await createFullBackupMetadata('backup1.json', 'backup1.xva')
await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
await vmBackupDir.init()
await vmBackupDir.clean({ remove: true })
const remainingFiles = await handler.list(rootPath)
assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
assert.ok(vmBackupDir.backupArchives.get(`/${rootPath}/backup1.json`).isValid)
assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept')
})
})

View File

@@ -1,168 +0,0 @@
import { basename, normalize } from '@xen-orchestra/fs/path'
import assert from 'node:assert'
import { FileDescriptor } from '@xen-orchestra/fs'
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
VmBackupInterface,
PartialBackupMetadata,
ResolvedBackupCleanOptions,
} from './VmBackup.types.mjs'
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
// https://tools.ietf.org/html/rfc1952.html#page-5
Buffer.from('1F8B', 'hex'),
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
Buffer.from('28B52FFD', 'hex'),
]
const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
return true
}
}
return false
}
const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
if (size <= 1024 || size % 512 !== 0) {
return false
}
const buf = Buffer.allocUnsafe(1024)
assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
return buf.every(byte => byte === 0)
}
export async function isValidXva(handler: RemoteHandlerAbstract, filePath: string): Promise<boolean> {
// size is longer when encrypted + reading part of an encrypted file is not implemented
if (handler.isEncrypted) {
return true
}
const fd = await handler.openFile(filePath, 'r')
try {
const size = await handler.getSize(filePath)
if (size < 20) {
// neither a valid gzip nor tar
return false
}
return (await isCompressedFile(handler, fd))
? true // compressed files cannot be validated at this time
: await isValidTar(handler, size, fd)
} finally {
handler.closeFile(fd).catch(noop)
}
}
const noop = (): void => {}
export class VmFullBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
rootPath: string
xvaPath: string
isValid?: boolean
missingDisk = false
metadataPath: string
metadata: PartialBackupMetadata
opts: ResolvedBackupCleanOptions
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
opts: ResolvedBackupCleanOptions
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.xvaPath = normalize(xvaPath)
this.opts = opts
}
async init() {}
/**
* As we are not 100% sure if XVA is valid,
* we prefer to warn if it is the case but not remove it.
*/
async check(): Promise<CheckResult> {
if (this.isValid === undefined) {
let fileSize = 0
let validDisk = false
let validSize = true
try {
fileSize = await this.handler.getSize(this.xvaPath)
validDisk = await isValidXva(this.handler, this.xvaPath)
} catch (error) {
validDisk = false
if (error?.code === 'ENOENT') {
this.missingDisk = true
}
this.opts.logWarn('Issue while checking XVA', { error })
}
try {
await this.handler.getSize(`${this.xvaPath}.checksum`)
} catch (error) {
this.opts.logWarn('Checksum file not valid, not blocking', { error })
}
if (this.metadata.size !== undefined) {
validSize = this.metadata.size === fileSize
}
this.isValid = fileSize > 0 && validSize && validDisk
}
if (!this.isValid) {
this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
}
return { isValid: this.isValid }
}
async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (this.isValid === undefined) {
await this.check()
}
const removedFiles: string[] = []
if (!this.isValid && this.missingDisk) {
removedFiles.push(this.metadataPath, `${this.xvaPath}.checksum`)
}
if (remove) {
if (!this.isValid) {
if (!this.missingDisk) {
this.opts.logWarn(`This files may be corrupted but not yet to be removed`, {
files: this.getAssociatedFiles({ prefix: false }),
})
} else {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
}
}
return { removedFiles, merge: false }
}
getAssociatedFiles({ prefix = false }): Array<string> {
let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
return prefix ? validFiles : validFiles.map(file => basename(file))
}
}

View File

@@ -1,688 +0,0 @@
import test from 'node:test'
import { strict as assert } from 'node:assert'
import tmp from 'tmp'
import fs from 'fs-extra'
import * as uuid from 'uuid'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
// eslint-disable-next-line n/no-missing-import
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
import { dirname, basename } from 'node:path'
import { rimraf } from 'rimraf'
const { beforeEach, afterEach, describe } = test
let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
const rootPath = 'xo-vm-backups/VMUUID/'
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
jobId = uniqueId()
vdiId = uniqueId()
relativePath = `vdis/${jobId}/${vdiId}`
basePath = `${rootPath}/${relativePath}`
vdiId2 = uniqueId()
relativePath2 = `vdis/${jobId}/${vdiId2}`
basePath2 = `${rootPath}/${relativePath2}`
await fs.mkdirp(`${tempDir}/${basePath}`)
await fs.mkdirp(`${tempDir}/${basePath2}`)
})
afterEach(async () => {
await rimraf(tempDir)
await handler.forget()
})
const uniqueId = () => uuid.v1()
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
async function generateVhd(path, opts = {}) {
let vhd
let dataPath = path
if (opts.useAlias) {
await handler.mkdir(dirname(path) + '/data/')
dataPath = dirname(path) + '/data/' + basename(path)
}
if (opts.mode === 'directory') {
await handler.mkdir(dataPath)
vhd = new VhdDirectory(handler, dataPath)
} else {
const fd = await handler.openFile(dataPath, 'wx')
vhd = new VhdFile(handler, fd)
}
vhd.header = { ...VHDHEADER, ...opts.header }
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
if (vhd.header.parentUuid) {
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
if (opts.useAlias === true) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
if (opts.blocks) {
for (const blockId of opts.blocks) {
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
}
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
return vhd
}
test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
// metadata references a non-existent disk so the lineage is created for this vdiDir
// and notReallyAVhd.vhd is treated as an unreferenced orphan
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
assert.ok(logged.includes('failed to open disk'))
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent, should be deleted
await generateVhd(`${basePath}/abandoned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// one orphan, which is a full vhd, no parent : should stay
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan in the metadata : should stay
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
})
)
// clean
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
// abandoned has a missing parent, orphan and child have no active backup reference
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
assert.equal(detectedMissingParent.length, 1)
// we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
})
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.json is not here
],
})
)
await generateVhd(`${basePath}/abandoned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
let logged = ''
const logInfo = message => {
logged += message + '\n'
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
let matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
// abandoned.vhd is not here anymore
],
}),
{ flags: 'w' }
)
logged = ''
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
matched = logged.match(/deleting unused disk/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after metadata deletion'
)
})
test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
// A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
// Metadata references the corrupt VHD as if it were a valid active disk
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/broken.vhd`],
})
)
let logged = ''
const logInfo = message => {
logged += message
}
await VmBackupDirectory.cleanVm(handler, rootPath, {
remove: true,
logInfo: () => {},
logWarn: logInfo,
})
assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
// metadata removed because backup is incomplete
assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
// broken VHD removed via the orphan path in lineage.clean()
assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
})
test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
// Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
// Each snapshot is the active disk for exactly one backup metadata file.
//
// Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
// own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
// RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
// their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
// deleted — destroying two live backups.
const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
})
await generateVhd(`${basePath}/snapshot3.vhd`, {
header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
})
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata3.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining = await handler.list(basePath)
assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
assert.equal(remaining.length, 3, 'no extra files should remain')
// All three metadata files must also survive (all backups are complete)
const rootFiles = await handler.list(rootPath)
assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
})
test('it handles multiple VDI directories independently', async () => {
// vdiDir1: diskA1 (base) <- diskA2, plus an orphan
const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
await generateVhd(`${basePath}/diskA2.vhd`, {
header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
})
await generateVhd(`${basePath}/orphanA.vhd`)
// vdiDir2: diskB1 (base) <- diskB2, plus an orphan
const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
await generateVhd(`${basePath2}/diskB2.vhd`, {
header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
})
await generateVhd(`${basePath2}/orphanB.vhd`)
// archive1 covers both base disks, archive2 covers both child disks
await handler.writeFile(
`${rootPath}/metadata1.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
)
await handler.writeFile(
`${rootPath}/metadata2.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
const remaining1 = await handler.list(basePath)
assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
const remaining2 = await handler.list(basePath2)
assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
})
test('it merges delta of non destroyed chain', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000, // a size too small
vhds: [
`${relativePath}/grandchild.vhd`, // grand child should not be merged
`${relativePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
blocks: [0, 1],
})
// a grand child
await generateVhd(`${basePath}/grandchild.vhd`, {
header: {
parentUnicodeName: 'child.vhd',
parentUuid: child.footer.uuid,
},
})
let logged = []
const logInfo = message => {
logged.push(message)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
assert.equal(logged[0], `unexpected number of entries in backup cache`)
logged = []
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
assert.equal(logged.includes(`merging disk chain`), true)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 2)
assert.equal(remainingVhds.includes('child.vhd'), true)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
assert.equal(metadata.size, 4299776, 'metadata.size should be updated after merge')
})
test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
// ancestor -> child -> grandchild
// only grandchild is referenced in metadata: both ancestor and child are orphans
// they must be merged as a single 3-disk chain, not processed separately
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000,
vhds: [`${relativePath}/grandchild.vhd`],
})
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
const child = await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
})
await generateVhd(`${basePath}/grandchild.vhd`, {
header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
})
const mergeChains = []
const logInfo = (message, opts) => {
if (message === 'merging disk chain') mergeChains.push(opts.chain)
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
// all three disks must be handled in exactly one merge, not two separate operations
assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
assert.equal(remainingVhds.includes('child.vhd'), false)
})
test('it resumes an interrupted merge with chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
)
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
blocks: [5, 6],
})
await generateVhd(`${basePath}/child.vhd`, {
header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
blocks: [7, 8],
})
await handler.writeFile(
`${basePath}/.ancestor.vhd.merge.json`,
JSON.stringify({
parent: { uuid: '0' },
child: { uuid: '0' },
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
currentBlock: 3,
mergedDataSize: 3 * 2 * 1024 * 1024,
step: 'mergeBlocks',
diskSize: 0,
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const files = await handler.list(basePath)
assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
})
test('it resumes an interrupted merge without chain field', async () => {
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
size: 209920,
vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUuid: orphan.footer.uuid,
},
})
// a merge in progress file
await handler.writeFile(
`${basePath}/.orphan.vhd.merge.json`,
JSON.stringify({
parent: {
header: orphan.header.checksum,
},
child: {
header: child.header.checksum,
},
})
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
assert.equal(remainingVhds.length, 1)
assert.equal(remainingVhds.includes('child.vhd'), true)
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message and resulting files should be identical to the output with vhd files which is tested independently
describe('tests multiple combination ', { concurrency: 1 }, () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
// a broken VHD
if (useAlias) {
await handler.mkdir(basePath + '/data')
}
const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
if (vhdMode === 'directory') {
await handler.mkdir(brokenVhdDataPath)
} else {
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
}
if (useAlias) {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metadata
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandoned delta vhd without its parent
await generateVhd(`${basePath}/abandoned.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'gone.vhd',
parentUuid: uniqueIdBuffer(),
},
})
// an ancestor of a vhd present in metadata
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
blocks: [1, 3],
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: ancestor.footer.uuid,
},
blocks: [1, 2],
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: child.footer.uuid,
},
blocks: [2, 3],
})
// an older parent that was merging in clean
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
useAlias,
mode: vhdMode,
})
// a clean vhd in metadata
const clean = await generateVhd(`${basePath}/clean.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUuid: cleanAncestor.footer.uuid,
},
})
await handler.writeFile(
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
JSON.stringify({
parent: {
header: cleanAncestor.header.checksum,
},
child: {
header: clean.header.checksum,
},
})
)
// the metadata file
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
`${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
`${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
],
})
)
if (!useAlias && vhdMode === 'directory') {
try {
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
} catch (err) {
assert.strictEqual(
err.code,
'NOT_SUPPORTED',
'Merging directory without alias should raise a not supported error'
)
return
}
assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
}
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 6501888)
// cache.json.gz must be regenerated when metadata changes after a merge
assert.ok(
(await handler.list(rootPath)).includes('cache.json.gz'),
'cache.json.gz should be regenerated after a merge'
)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
if (useAlias) {
const dataSurvivors = await handler.list(basePath + '/data')
// the goal of the alias : do not move a full folder
assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
assert.equal(survivors.length, 4) // the 3 ok + data
assert.equal(dataSurvivors.length, 3)
} else {
assert.equal(survivors.includes('clean.vhd'), true)
assert.equal(survivors.includes('child.vhd'), true)
assert.equal(survivors.includes('grandchild.vhd'), true)
assert.equal(survivors.length, 3)
}
})
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
// metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
assert.deepEqual(await handler.list(basePath), [])
})
test('check all types of aliases, corrupted, missing or not', async () => {
// valid alias pointing to an existing data file
await handler.mkdir(`${basePath}/data`)
await generateVhd(`${basePath}/data/ok.vhd`)
await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
// broken alias pointing to a missing data file
await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
// alias pointing to an existing but corrupt data file
await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
// data file without alias
await generateVhd(`${basePath}/data/missingalias.vhd`)
// only the valid alias is referenced by a complete backup
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
)
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
// only ok.alias.vhd has survived
const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
assert.equal(aliases.length, 1)
assert.equal(aliases[0], 'ok.alias.vhd')
// corrupt.vhd and missingalias.vhd must be gone
const data = await handler.list(`${basePath}/data`)
assert.equal(data.length, 1)
assert.equal(data[0], 'ok.vhd')
})

View File

@@ -1,174 +0,0 @@
import {
ArchiveCleanOptions,
CheckResult,
CleanResult,
ResolvedBackupCleanOptions,
VmBackupInterface,
PartialBackupMetadata,
} from './VmBackup.types.mjs'
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
export class VmIncrementalBackupArchive implements VmBackupInterface {
handler: RemoteHandlerAbstract
metadataPath: string
metadata: PartialBackupMetadata
readonly diskPaths: Array<string>
rootPath: string
opts: ResolvedBackupCleanOptions
diskLineages: Map<string, RemoteDiskLineage> = new Map()
#isChecked = false
#isComplete = false
get isChecked(): boolean {
return this.#isChecked
}
get isComplete(): boolean {
return this.#isComplete
}
constructor(
handler: RemoteHandlerAbstract,
rootPath: string, // xo-vm-backups/<vmUuid>/
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
metadata: PartialBackupMetadata,
diskPaths: Array<string>, // xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/<snapshotUuid>.alias.vhd (one per VDI)
opts: ResolvedBackupCleanOptions,
lineageRegistry?: Map<string, RemoteDiskLineage>
) {
this.handler = handler
this.rootPath = normalize(rootPath)
this.metadataPath = normalize(metadataPath)
this.metadata = metadata
this.diskPaths = diskPaths.map(path => normalize(path))
this.opts = opts
if (lineageRegistry !== undefined) {
this.diskLineages = lineageRegistry
}
}
async init(): Promise<void> {
// Build one RemoteDiskLineage per VDI directory derived from diskPaths
try {
for (const diskPath of this.diskPaths) {
const vdiDir = dirname(diskPath)
if (!this.diskLineages.has(vdiDir)) {
const lineage = new RemoteDiskLineage(this.handler, vdiDir, this.opts)
await lineage.init()
this.diskLineages.set(vdiDir, lineage)
}
}
} catch (error) {
if (error?.code === 'NOT_SUPPORTED') throw error
this.opts.logWarn('failed to scan VDI directories', { error })
}
}
async check(): Promise<CheckResult> {
const missingDisks: string[] = []
const brokenDisks: string[] = []
if (this.diskPaths.length === 0) {
this.opts.logWarn('incremental backup has no disk paths', { metadataPath: this.metadataPath })
this.#isComplete = false
} else {
for (const diskPath of this.diskPaths) {
const lineage = this.diskLineages.get(dirname(diskPath))
if (lineage === undefined || !lineage.isPathAccessible(diskPath)) {
if (lineage?.isBroken(diskPath)) {
this.opts.logWarn('disk check error', { path: diskPath })
brokenDisks.push(diskPath)
} else {
this.opts.logWarn('disk file missing', { path: diskPath })
missingDisks.push(diskPath)
}
}
}
this.#isComplete = missingDisks.length === 0 && brokenDisks.length === 0
if (!this.#isComplete) {
this.opts.logWarn('incremental backup is incomplete', {
metadataPath: this.metadataPath,
missingDisks,
brokenDisks,
})
}
}
if (this.#isComplete) {
for (const diskPath of this.diskPaths) {
this.diskLineages.get(dirname(diskPath))?.addActiveDiskPath(diskPath)
}
}
this.#isChecked = true
return { isValid: this.#isComplete, missingDisks, brokenDisks }
}
/**
* Removes the metadata file if the backup is incomplete (missing disks).
* If mergedSizes is provided, updates metadata with the total merged size for this archive's disks.
* Actual disk merge/deletion is handled by RemoteDiskLineage.
*/
async clean({ remove = this.opts.remove ?? false, mergedSizes }: ArchiveCleanOptions = {}): Promise<CleanResult> {
if (!this.#isChecked) {
await this.check()
}
const removedFiles: string[] = []
if (!this.#isComplete) {
removedFiles.push(this.metadataPath)
}
if (remove) {
for (const file of removedFiles) {
try {
await this.handler.unlink(file)
} catch (error) {
this.opts.logWarn(`Issue removing ${file}`, { error })
}
}
}
let mergedSize = 0
if (mergedSizes !== undefined) {
for (const diskPath of this.diskPaths) {
mergedSize += mergedSizes.get(diskPath) ?? 0
}
if (mergedSize > 0) {
await this.updateMetadata(mergedSize)
}
}
return { removedFiles }
}
/**
* Updates metadata.size after a merge and writes the updated metadata file to disk.
* Also resets isVhdDifferencing so all VDIs are marked as full (non-differencing).
*/
async updateMetadata(mergedSize: number): Promise<void> {
this.metadata.size = mergedSize
if (this.metadata.vdis !== undefined) {
this.metadata.isVhdDifferencing = {}
for (const id of Object.keys(this.metadata.vdis)) {
this.metadata.isVhdDifferencing[id] = false
}
}
await this.handler.writeFile(this.metadataPath, JSON.stringify(this.metadata), { flags: 'w' })
}
/**
* Returns the metadata file path and active disk paths if the backup is complete, empty otherwise.
* Must be called after check().
*/
getAssociatedFiles({ prefix = false }): Array<string> {
if (!this.#isComplete) {
return []
}
const files = [this.metadataPath, ...this.diskPaths]
return prefix ? files : files.map(file => basename(file))
}
}

View File

@@ -1,108 +0,0 @@
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
export { RemoteDisk } from './RemoteDisk.mjs'
export { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
export { openDiskChain } from './openDiskChain.mjs'
export { MergeRemoteDisk } from './MergeRemoteDisk.mjs'
const DISK_EXTENSIONS = ['.vhd']
/**
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
* @typedef {import('./MergeRemoteDisk.mjs').MergeState} MergeState
*/
export function isDisk(path) {
return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
}
/**
*
* @param {Object} params
* @param {RemoteHandlerAbstract} params.handler
* @param {string} params.path
* @param {boolean} [params.force]
* @param {boolean} [params.ignoreBlockIndexes]
* @returns {Promise<Disposable<RemoteDisk>>}
*/
export async function openDisposableDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
const disk = new RemoteVhdDisk({ handler, path })
if (ignoreBlockIndexes) {
// Best-effort init: return the disk even if init fails so that callers
// can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
} else {
await disk.init({ force })
}
return {
value: disk,
dispose: () => disk.close(),
}
}
/**
* @param {Object} params
* @param {RemoteHandlerAbstract} params.handler
* @param {string} params.path
* @param {boolean} [params.force]
* @param {boolean} [params.ignoreBlockIndexes]
* @returns {Promise<RemoteDisk>}
*/
export async function openDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
const disk = new RemoteVhdDisk({ handler, path })
if (ignoreBlockIndexes) {
// Best-effort init: return the disk even if init fails so that callers
// can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
} else {
await disk.init({ force })
}
return disk
}
/**
* Deletes VDI directories under `vmDir/vdis/<jobId>/<vdiId>/` that are not in coveredDirs.
* Used to clean up disk directories that no backup metadata references at all.
*
* @param {RemoteHandlerAbstract} handler
* @param {string} vmDir - root of the VM backup directory (xo-vm-backups/<vmUuid>)
* @param {Set<string>} coveredDirs - normalized vdiDir paths already managed by a lineage
* @param {{ remove?: boolean, logWarn?: Function, logInfo?: Function }} opts
*/
export async function cleanOrphanDiskDirs(
handler,
vmDir,
coveredDirs,
{ remove = false, logWarn = () => {}, logInfo = () => {} } = {}
) {
const vdisPath = `${vmDir}/vdis`
let jobDirs
try {
jobDirs = await handler.list(vdisPath, { prependDir: true })
} catch {
return
}
for (const jobDir of jobDirs) {
let vdiDirs
try {
vdiDirs = await handler.list(jobDir, { prependDir: true })
} catch {
continue
}
for (const vdiDir of vdiDirs) {
if (!coveredDirs.has(vdiDir)) {
logWarn('orphan VDI directory, not referenced by any backup', { path: vdiDir })
if (remove) {
logInfo('deleting orphan VDI directory', { path: vdiDir })
try {
await handler.rmtree(vdiDir)
} catch (error) {
logWarn('failed to delete orphan VDI directory', { path: vdiDir, error })
}
}
}
}
}
}

View File

@@ -1,54 +0,0 @@
// @ts-check
/**
*
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
*/
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
import { defer } from 'golike-defer'
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
/**
* @param {any} $defer
* @param {Object} params
* @param {RemoteHandlerAbstract} params.handler
* @param {string} params.path
* @param {string | undefined} params.until
* @param {boolean} [params.force]
* @returns {Promise<RemoteDisk>}
*/
async function _openDiskChain($defer, { handler, path, until, force = false }) {
/**
* @type {RemoteVhdDisk}
*/
let disk
/**
* @type {Array<RemoteVhdDisk>}
*/
const disks = []
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
disk = new RemoteVhdDisk({ handler, path })
await disk.init({ force })
disks.push(disk)
let foundRootDisk = until === undefined
while (disk.isDifferencing()) {
disk = await disk.openParent()
if (disk.getPath() === until) {
foundRootDisk = true
await disk.close()
break
}
disks.unshift(disk)
}
if (!foundRootDisk) {
throw new Error(`trying to open chain from ${path} to ${until}, root disk not found`)
}
// the root disk
return new RemoteVhdDiskChain({ disks })
}
/**
* @type {(params: { handler: RemoteHandlerAbstract, path: string, until?: string, force?: boolean }) => Promise<RemoteDisk>}
*/
export const openDiskChain = defer(_openDiskChain)

View File

@@ -1 +0,0 @@
export { VmBackupDirectory } from './VmBackupDirectory.mjs'

View File

@@ -1,92 +0,0 @@
// a valid footer of a 2
export const VHDFOOTER = {
cookie: 'conectix',
features: 2,
fileFormatVersion: 65536,
dataOffset: 512,
timestamp: 0,
creatorApplication: 'caml',
creatorVersion: 1,
creatorHostOs: 0,
originalSize: 53687091200,
currentSize: 53687091200,
diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
diskType: 3,
checksum: 4294962945,
uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
saved: '',
hidden: '',
reserved: '',
}
export const VHDHEADER = {
cookie: 'cxsparse',
dataOffset: undefined,
tableOffset: 2048,
headerVersion: 65536,
maxTableEntries: 25600,
blockSize: 2097152,
checksum: 4294964241,
parentUuid: null,
parentTimestamp: 0,
reserved1: 0,
parentUnicodeName: '',
parentLocatorEntry: [
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
],
reserved2: '',
}

View File

@@ -1,18 +0,0 @@
{
"compilerOptions": {
"outDir": "./dist",
"declaration": true,
"strict": true,
"module": "nodenext",
"moduleResolution": "nodenext",
"target": "ES2022",
"rootDir": "./src",
"skipLibCheck": true,
"noImplicitAny": false,
"allowJs": true,
"useUnknownInCatchVariables": false,
"types": ["node"] // Only use types from Node.js and packages that are explicitly imported
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

View File

@@ -19,7 +19,7 @@ async function cleanFiles(vmDirs, { fix, merge, remove }) {
try {
nbStarted++
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
fix,
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),
@@ -50,7 +50,7 @@ async function cleanRemote(url, { fix, merge, remove }) {
try {
nbStarted++
await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
fix,
fixMetadata: fix,
remove,
merge,
logInfo: (...args) => console.log(...args),

View File

@@ -8,7 +8,7 @@
"dependencies": {
"@vates/async-each": "^1.0.3",
"@xen-orchestra/async-map": "^0.1.3",
"@xen-orchestra/backups": "^0.72.1",
"@xen-orchestra/backups": "^0.72.0",
"@xen-orchestra/fs": "^4.8.0",
"filenamify": "^6.0.0",
"getopts": "^2.2.5",

View File

@@ -10,7 +10,7 @@ import { dirname, join } from 'node:path'
import pickBy from 'lodash/pickBy.js'
import { defer } from 'golike-defer'
import { NegativeDisk } from '@xen-orchestra/disk-transform'
import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
import { openDiskChain } from './disks/openDiskChain.mjs'
import { resetVmOtherConfig } from './_otherConfig.mjs'
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')

View File

@@ -22,7 +22,7 @@ import * as tar from 'tar'
import zlib from 'zlib'
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
import { cleanVm } from './_cleanVm.mjs'
import { formatFilenameDate } from './_filenameDate.mjs'
import { getTmpDir } from './_getTmpDir.mjs'
import { isMetadataFile } from './_backupType.mjs'
@@ -31,7 +31,8 @@ import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from '
import { lvs, pvs } from './_lvm.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
import { openDiskChain } from './disks/openDiskChain.mjs'
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
import { ReadAhead } from '@xen-orchestra/disk-transform'
@@ -701,7 +702,7 @@ export class RemoteAdapter {
const handler = this._handler
if (this.useVhdDirectory()) {
return await writeToVhdDirectory({
await writeToVhdDirectory({
disk,
target: {
handler,
@@ -713,9 +714,8 @@ export class RemoteAdapter {
})
} else {
const stream = await toVhdStream(disk)
const size = await this.outputStream(path, stream, { validator, checksum: false })
await this.outputStream(path, stream, { validator, checksum: false })
await validator(path)
return size
}
}
@@ -878,14 +878,11 @@ export class RemoteAdapter {
}
Object.assign(RemoteAdapter.prototype, {
cleanVm(vmBackupPath, opts = {}) {
const { lock = true, ...cleanOpts } = opts
cleanVm(vmDir, { lock = true } = {}) {
if (lock) {
return Disposable.use(this._handler.lock(vmBackupPath), () => {
return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
})
return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
} else {
return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
return cleanVm.apply(this, arguments)
}
},
isValidXva,

View File

@@ -9,6 +9,7 @@ import { pFromCallback } from 'promise-toolbox'
import { RemoteAdapter } from './RemoteAdapter.mjs'
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
import { checkAliases } from './_cleanVm.mjs'
import { dirname, basename } from 'node:path'
import { rimraf } from 'rimraf'
@@ -81,17 +82,13 @@ test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/notReallyAVhd.vhd`] })
)
assert.equal((await handler.list(basePath)).length, 1)
let logged = ''
const logInfo = message => {
logged += message
}
await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
assert.ok(logged.includes('disk check error'), `expected 'disk check error' in: ${logged}`)
assert.equal(logged, `VHD check error`)
// not removed
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
// really remove it
@@ -118,10 +115,10 @@ test('it remove vhd with missing or multiple ancestors', async () => {
},
})
await handler.writeFile(
`${rootPath}/metadata.json`,
`metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
}),
{ flags: 'w' }
)
@@ -132,13 +129,13 @@ test('it remove vhd with missing or multiple ancestors', async () => {
}
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
assert.equal(detectedMissingParent.length, 1) // only one vhd should have been deleted
const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
})
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`${rootPath}/metadata.json`,
@@ -170,7 +167,7 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
logged += message + '\n'
}
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
let matched = logged.match(/deleting unused disk/g) || []
let matched = logged.match(/deleting unused VHD/g) || []
assert.equal(matched.length, 1) // only one vhd should have been deleted
// a missing vhd cause clean to remove all vhds
@@ -179,9 +176,9 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
JSON.stringify({
mode: 'delta',
vhds: [
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
`${relativePath}/orphan.vhd`,
`${relativePath}/child.vhd`,
`deleted.vhd`, // in metadata but not in vhds
`orphan.vhd`,
`child.vhd`,
// abandoned.vhd is not here anymore
],
}),
@@ -189,7 +186,7 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
)
logged = ''
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
matched = logged.match(/deleting unused disk/g) || []
matched = logged.match(/deleting unused VHD/g) || []
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
})
@@ -236,7 +233,7 @@ test('it merges delta of non destroyed chain', async () => {
const result = await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
assert.ok(result.size > 0, `no merged size`)
const [merging] = logged
assert.equal(merging, `Disk chain needs merging`)
assert.equal(merging, `merging VHD chain`)
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size should be the size of children + grand children after the merge
@@ -454,8 +451,8 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
await adapter.cleanVm(rootPath, { remove: true, merge: true, logWarn: () => {}, lock: false })
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
// size is the sum of the merged chains' final on-disk sizes:
assert.deepEqual(metadata.size, 6501888)
// size should be the size of children + grand children + clean after the merge
assert.deepEqual(metadata.size, 4404224)
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
// ancestor and child should be merged
@@ -484,12 +481,32 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
await handler.writeFile(
`${rootPath}/metadata.json`,
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
)
await adapter.cleanVm(rootPath, { remove: true, logWarn: () => {}, lock: false })
assert.deepEqual(await handler.list(basePath), [])
})
test('check Aliases should work alone', async () => {
await handler.mkdir('vhds')
await handler.mkdir('vhds/data')
await generateVhd(`vhds/data/ok.vhd`)
await VhdAbstract.createAlias(handler, 'vhds/ok.alias.vhd', 'vhds/data/ok.vhd')
await VhdAbstract.createAlias(handler, 'vhds/missingData.alias.vhd', 'vhds/data/nonexistent.vhd')
await generateVhd(`vhds/data/missingalias.vhd`)
await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', {
remove: true,
handler,
logWarn: () => {},
})
// only ok have survived
const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
assert.equal(alias.length, 1)
const data = await handler.list('vhds/data')
assert.equal(data.length, 1)
})

View File

@@ -0,0 +1,628 @@
import * as UUID from 'uuid'
import { asyncMap } from '@xen-orchestra/async-map'
import { Constants, openVhd, VhdAbstract } from 'vhd-lib'
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
import { basename, dirname, resolve } from 'node:path'
import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
import { limitConcurrency } from 'limit-concurrency-decorator'
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
import { Disposable } from 'promise-toolbox'
import { Task } from '@vates/task'
import handlerPath from '@xen-orchestra/fs/path'
const { DISK_TYPES } = Constants
// chain is [ ancestor, child_1, ..., child_n ]
async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
logInfo(`merging VHD chain`, { chain })
let done, total
const handle = setInterval(() => {
if (done !== undefined) {
logInfo('merge in progress', {
done,
parent: chain[0],
progress: Math.round((100 * done) / total),
total,
})
}
}, 10e3)
try {
const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
const childDisks = []
for (const path of chain) {
childDisks.push(new RemoteVhdDisk({ handler, path }))
}
const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
logInfo,
mergeBlockConcurrency,
onProgress({ done: d, total: t }) {
done = d
total = t
},
removeUnused: remove,
})
const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
await parentDisk.init({ force: isResumingMerge })
await childDiskChain.init({ force: isResumingMerge })
const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
return result
} finally {
clearInterval(handle)
}
}
const noop = Function.prototype
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
const listVhds = async (handler, vmDir, logWarn) => {
const vhds = new Set()
const aliases = {}
const interruptedVhds = new Map()
const checksums = new Set()
await asyncMap(
await handler.list(`${vmDir}/vdis`, {
ignoreMissing: true,
prependDir: true,
}),
async jobDir =>
asyncMap(
await handler.list(jobDir, {
prependDir: true,
}),
async vdiDir => {
const list = await handler.list(vdiDir, {
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
})
aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
await asyncMap(list, async file => {
if (isVhdSumFile(file)) {
checksums.add(`${vdiDir}/${file}`)
return
}
const res = INTERRUPTED_VHDS_REG.exec(file)
if (res === null) {
vhds.add(`${vdiDir}/${file}`)
} else {
try {
const mergeState = JSON.parse(await handler.readFile(`${vdiDir}/${file}`))
interruptedVhds.set(`${vdiDir}/${res[1]}`, {
statePath: `${vdiDir}/${file}`,
chain: mergeState.chain,
})
} catch (error) {
// fall back to a non resuming merge
vhds.add(`${vdiDir}/${file}`)
logWarn('failed to read existing merge state', { path: file, error })
}
}
})
}
)
)
return { vhds, interruptedVhds, aliases, checksums }
}
export async function checkAliases(
aliasPaths,
targetDataRepository,
{ handler, logInfo = noop, logWarn = console.warn, remove = false }
) {
const aliasFound = []
for (const alias of aliasPaths) {
let target
try {
target = await resolveVhdAlias(handler, alias)
} catch (err) {
if (err.code === 'ENOENT') {
logWarn('missing target of alias', { alias })
if (remove) {
logInfo('removing alias and non VHD target', { alias, target })
await handler.unlink(alias)
}
continue
}
if (err.code === 'EISDIR') {
logWarn('alias is a vhd directory', { alias })
if (remove) {
logInfo('removing vhd directory named as alias', { alias, target })
await VhdAbstract.unlink(handler, alias)
}
continue
}
logWarn('unhandled error while checking alias', { alias, err })
continue
}
if (target === '') {
logWarn('empty target for alias ', { alias })
if (remove) {
logInfo('removing alias and non VHD target', { alias, target })
await handler.unlink(alias)
}
continue
}
if (!isVhdFile(target)) {
logWarn('alias references non VHD target', { alias, target })
if (remove) {
logInfo('removing alias and non VHD target', { alias, target })
await handler.unlink(target)
await handler.unlink(alias)
}
continue
}
try {
const { dispose } = await openVhd(handler, target)
try {
await dispose()
} catch (e) {
// error during dispose should not trigger a deletion
}
} catch (error) {
logWarn('missing or broken alias target', { alias, target, error })
if (remove) {
try {
await VhdAbstract.unlink(handler, alias)
} catch (error) {
if (error.code !== 'ENOENT') {
logWarn('error deleting alias target', { alias, target, error })
}
}
}
continue
}
aliasFound.push(resolve('/', target))
}
const vhds = await handler.list(targetDataRepository, {
ignoreMissing: true,
prependDir: true,
})
await asyncMap(vhds, async path => {
if (!aliasFound.includes(path)) {
logWarn('no alias references VHD', { path })
if (remove) {
logInfo('deleting unused VHD', { path })
await VhdAbstract.unlink(handler, path)
}
}
})
}
const defaultMergeLimiter = limitConcurrency(1)
export async function cleanVm(
vmDir,
{
fixMetadata,
remove = false,
removeTmp = remove,
merge,
mergeBlockConcurrency,
mergeLimiter = defaultMergeLimiter,
logInfo = noop,
logWarn = console.warn,
}
) {
const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
const handler = this._handler
const vhdsToJSons = new Set()
const vhdById = new Map()
const vhdParents = { __proto__: null }
const vhdChildren = { __proto__: null }
const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
// from 5.110 to 5.113 we computed checksum for vhd file
// but never used nor removed them
await asyncMap(checksums, async path => {
if (remove) {
logInfo('deleting checksum file ', { path })
return handler.unlink(path)
}
})
// remove broken VHDs
await asyncMap(vhds, async path => {
if (removeTmp && basename(path)[0] === '.') {
logInfo('deleting temporary VHD', { path })
return VhdAbstract.unlink(handler, path)
}
try {
await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
vhdParents[path] = parent
if (parent in vhdChildren) {
const error = new Error('this script does not support multiple VHD children')
error.parent = parent
error.child1 = vhdChildren[parent]
error.child2 = path
throw error // should we throw?
}
vhdChildren[parent] = path
}
// Detect VHDs with the same UUIDs
//
// Due to a bug introduced in a1bcd35e2
const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
let vhdKept = vhd
if (duplicate !== undefined) {
logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
if (duplicate.containsAllDataOf(vhd)) {
logWarn(`should delete ${path}`)
vhdKept = duplicate
vhds.delete(path)
} else if (vhd.containsAllDataOf(duplicate)) {
logWarn(`should delete ${duplicate._path}`)
vhds.delete(duplicate._path)
} else {
logWarn('same ids but different content')
}
}
vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
})
} catch (error) {
vhds.delete(path)
logWarn('VHD check error', { path, error })
if (error?.code === 'ERR_ASSERTION' && remove) {
logInfo('deleting broken VHD', { path })
return VhdAbstract.unlink(handler, path)
}
}
})
// remove interrupted merge states for missing VHDs
for (const interruptedVhd of interruptedVhds.keys()) {
if (!vhds.has(interruptedVhd)) {
const { statePath } = interruptedVhds.get(interruptedVhd)
interruptedVhds.delete(interruptedVhd)
logWarn('orphan merge state', {
mergeStatePath: statePath,
missingVhdPath: interruptedVhd,
})
if (remove) {
logInfo('deleting orphan merge state', { statePath })
await handler.unlink(statePath)
}
}
}
// check if alias are correct
// check if all vhd in data subfolder have a corresponding alias
await asyncMap(Object.keys(aliases), async dir => {
await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
})
// remove VHDs with missing ancestors
{
const deletions = []
// return true if the VHD has been deleted or is missing
const deleteIfOrphan = vhdPath => {
const parent = vhdParents[vhdPath]
if (parent === undefined) {
return
}
// no longer needs to be checked
delete vhdParents[vhdPath]
deleteIfOrphan(parent)
if (!vhds.has(parent)) {
vhds.delete(vhdPath)
logWarn('parent VHD is missing', { parent, child: vhdPath })
if (remove) {
logInfo('deleting orphan VHD', { path: vhdPath })
deletions.push(VhdAbstract.unlink(handler, vhdPath))
}
}
}
// > A property that is deleted before it has been visited will not be
// > visited later.
// >
// > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
for (const child in vhdParents) {
deleteIfOrphan(child)
}
await Promise.all(deletions)
}
const jsons = new Set()
const xvas = new Set()
const xvaSums = []
const entries = await handler.list(vmDir, {
prependDir: true,
})
entries.forEach(path => {
if (isMetadataFile(path)) {
jsons.add(path)
} else if (isXvaFile(path)) {
xvas.add(path)
} else if (isXvaSumFile(path)) {
xvaSums.push(path)
}
})
const cachePath = vmDir + '/cache.json.gz'
let mustRegenerateCache
{
const cache = await this._readCache(cachePath)
const actual = cache === undefined ? 0 : Object.keys(cache).length
const expected = jsons.size
mustRegenerateCache = actual !== expected
if (mustRegenerateCache) {
logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
}
}
await asyncMap(xvas, async path => {
// check is not good enough to delete the file, the best we can do is report
// it
if (!(await this.isValidXva(path))) {
logWarn('XVA might be broken', { path })
}
})
const unusedVhds = new Set(vhds)
const unusedXvas = new Set(xvas)
const backups = new Map()
// compile the list of unused XVAs and VHDs, and remove backup metadata which
// reference a missing XVA/VHD
await asyncMap(jsons, async json => {
let metadata
try {
metadata = JSON.parse(await handler.readFile(json))
} catch (error) {
logWarn('failed to read backup metadata', { path: json, error })
jsons.delete(json)
return
}
let isBackupComplete
const { mode } = metadata
if (mode === 'full') {
const linkedXva = resolve('/', vmDir, metadata.xva)
isBackupComplete = xvas.has(linkedXva)
if (isBackupComplete) {
unusedXvas.delete(linkedXva)
} else {
logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
}
} else if (mode === 'delta') {
const linkedVhds = (() => {
const { vhds } = metadata
return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
})()
const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
isBackupComplete = missingVhds.length === 0
// FIXME: find better approach by keeping as much of the backup as
// possible (existing disks) even if one disk is missing
if (isBackupComplete) {
linkedVhds.forEach(_ => unusedVhds.delete(_))
linkedVhds.forEach(path => {
vhdsToJSons[path] = json
})
} else {
logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
}
}
if (isBackupComplete) {
backups.set(json, metadata)
} else {
jsons.delete(json)
if (remove) {
logInfo('deleting incomplete backup', { backup: json })
mustRegenerateCache = true
await handler.unlink(json)
}
}
})
// TODO: parallelize by vm/job/vdi
const unusedVhdsDeletion = []
const toMerge = []
{
// VHD chains (as list from oldest to most recent) to merge indexed by most recent
// ancestor
const vhdChainsToMerge = { __proto__: null }
const toCheck = new Set(unusedVhds)
const getUsedChildChainOrDelete = vhd => {
if (vhd in vhdChainsToMerge) {
const chain = vhdChainsToMerge[vhd]
delete vhdChainsToMerge[vhd]
return chain
}
if (!unusedVhds.has(vhd)) {
return [vhd]
}
// no longer needs to be checked
toCheck.delete(vhd)
const child = vhdChildren[vhd]
if (child !== undefined) {
const chain = getUsedChildChainOrDelete(child)
if (chain !== undefined) {
chain.unshift(vhd)
return chain
}
}
// no warning because a VHD can be unused for perfectly good reasons,
// e.g. the corresponding backup (metadata file) has been deleted
if (remove) {
logInfo('deleting unused VHD', { path: vhd })
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
}
}
toCheck.forEach(vhd => {
vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
})
// merge interrupted VHDs
for (const parent of interruptedVhds.keys()) {
// before #6349 the chain wasn't in the mergeState
const { chain, statePath } = interruptedVhds.get(parent)
if (chain === undefined) {
vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
} else {
vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
}
}
Object.values(vhdChainsToMerge).forEach(chain => {
if (chain !== undefined) {
toMerge.push(chain)
}
})
}
let totalMergedDataSize = 0
const metadataWithMergedVhd = {}
const doMerge = async () => {
await asyncMap(toMerge, async chain => {
const { finalDiskSize, mergedDataSize } = await limitedMergeVhdChain(handler, chain, {
logInfo,
logWarn,
remove,
mergeBlockConcurrency,
})
totalMergedDataSize += mergedDataSize
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
})
return { size: totalMergedDataSize }
}
await Promise.all([
...unusedVhdsDeletion,
toMerge.length !== 0 && (merge ? Task.run({ properties: { name: 'merge' } }, doMerge) : () => Promise.resolve()),
asyncMap(unusedXvas, path => {
logWarn('unused XVA', { path })
if (remove) {
logInfo('deleting unused XVA', { path })
return handler.unlink(path)
}
}),
asyncMap(xvaSums, path => {
// no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
if (!xvas.has(path.slice(0, -'.checksum'.length))) {
logInfo('unused XVA checksum', { path })
if (remove) {
logInfo('deleting unused XVA checksum', { path })
return handler.unlink(path)
}
}
}),
])
// update size for delta metadata with merged VHD
// check for the other that the size is the same as the real file size
await asyncMap(jsons, async metadataPath => {
const metadata = backups.get(metadataPath)
let fileSystemSize
const mergedSize = metadataWithMergedVhd[metadataPath]
const { mode, size, xva } = metadata
try {
if (mode === 'full') {
// a full backup : check size
const linkedXva = resolve('/', vmDir, xva)
try {
fileSystemSize = await handler.getSize(linkedXva)
if (fileSystemSize !== size && fileSystemSize !== undefined) {
logWarn('cleanVm: incorrect backup size in metadata', {
path: metadataPath,
actual: size ?? 'none',
expected: fileSystemSize,
})
}
} catch (error) {
// will fail with encrypted remote
}
}
} catch (error) {
logWarn('failed to get backup size', { backup: metadataPath, error })
return
}
// systematically update size and differentials after a merge
// @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
metadata.size = mergedSize ?? fileSystemSize ?? size
if (mergedSize) {
// all disks are now key disk
metadata.isVhdDifferencing = {}
for (const id of Object.keys(metadata.vdis ?? {})) {
metadata.isVhdDifferencing[id] = false
}
}
mustRegenerateCache = true
try {
await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
} catch (error) {
logWarn('failed to update backup size in metadata', { path: metadataPath, error })
}
}
})
if (mustRegenerateCache) {
const cache = {}
for (const [path, content] of backups.entries()) {
cache[path] = {
_filename: path,
id: path,
...content,
}
}
await this._writeCache(cachePath, cache)
}
return {
// boolean whether some VHDs were merged (or should be merged)
merge: toMerge.length !== 0,
size: totalMergedDataSize,
}
}

View File

@@ -85,7 +85,6 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
export function resetVmOtherConfig(xapi, vmRef) {
return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
return xapi.setFieldEntries(type, ref, 'other_config', {
[CONTENT_KEY]: null,
[COPY_OF]: null,
[DATETIME]: null,
[DELTA_CHAIN_LENGTH]: null,
@@ -93,7 +92,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
[JOB_ID]: null,
[SCHEDULE_ID]: null,
[VM_UUID]: null,
// REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
})
})

View File

@@ -71,7 +71,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
const queue = new Set(vmsUuids)
const taskByVmId = {}
const nTriesByVmId = {}
const vmErrors = []
const handleVm = vmUuid => {
if (nTriesByVmId[vmUuid] === undefined) {
@@ -127,7 +126,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
return task.success(result)
}
if (isLastRun) {
vmErrors.push(taskError)
return task.failure(taskError)
}
// don't end the task
@@ -150,8 +148,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
await asyncMapSettled(vmIds, _handleVm)
}
this._throwVmErrors(vmErrors)
}
)
}

View File

@@ -98,7 +98,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
const queue = new Set(vmIds)
const taskByVmId = {}
const nTriesByVmId = {}
const vmErrors = []
const handleVm = vmUuid => {
const getVmTask = () => {
@@ -176,19 +175,11 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
} else {
// ending the task with error or not ending the task
vmBackupFailed(taskError, task)
if (isLastRun) {
vmErrors.push(taskError)
}
}
})
.catch(noop) // errors are handled by logs
}),
error => {
vmBackupFailed(error, getVmTask())
if (isLastRun) {
vmErrors.push(error)
}
}
error => vmBackupFailed(error, getVmTask())
)
}
const { concurrency } = settings
@@ -200,8 +191,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
await asyncMapSettled(vmIds, _handleVm)
}
this._throwVmErrors(vmErrors)
}
)
}

View File

@@ -11,22 +11,7 @@ export const DEFAULT_SETTINGS = {
reportWhen: 'failure',
}
export class AggregateError extends Error {
constructor(errors, message) {
super(message)
this.errors = errors
}
}
export const Abstract = class AbstractRunner {
_throwVmErrors(errors) {
if (errors.length === 1) {
throw errors[0]
} else if (errors.length > 1) {
throw new AggregateError(errors, `${errors.length} VMs failed`)
}
}
constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
this._config = config
this._getRecord = getConnectedRecord

View File

@@ -8,7 +8,6 @@ import { exportIncrementalVm } from '../../_incrementalVm.mjs'
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
import {
CONTENT_KEY,
DATETIME,
DELTA_CHAIN_LENGTH,
EXPORTED_SUCCESSFULLY,
@@ -102,136 +101,63 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
}
/**
* Groups snapshot VDIs by their DATETIME other_config value and sorts them most-recent-first.
*
* @param {import('@vates/types').XenApiVdi[]} snapshotVdis
* @returns {{ datetime: string, vdis: import('@vates/types').XenApiVdi[] }[]}
*/
_groupSnapshotVdisByDatetime(snapshotVdis) {
const byDatetime = new Map()
for (const vdi of snapshotVdis) {
const datetime = vdi.other_config[DATETIME]
if (!byDatetime.has(datetime)) {
byDatetime.set(datetime, [])
}
byDatetime.get(datetime).push(vdi)
}
// sort them by decreasing datetime
return [...byDatetime.entries()]
.sort(([a], [b]) => b.localeCompare(a))
.map(([datetime, vdis]) => ({ datetime, vdis }))
}
async _selectBaseVm() {
const xapi = this._xapi
// filter _jobSnapshotVdis to have only the last successfully exported vdi
// compute delta chain length
// fill baseUuidToSrcVdi with this data
const exportedVdis = this._jobSnapshotVdis.filter(
_ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
)
let lastSuccessfullBackup
let lastExportedVdis = []
for (const exportedVdi of exportedVdis) {
if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
lastExportedVdis = []
lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
}
if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
lastExportedVdis.push(exportedVdi)
}
}
this._baseVdis = {}
if (lastExportedVdis.length === 0) {
debug('no base VDIS found', {
jobLength: this._jobSnapshotVdis.length,
lastSuccessfullBackup,
exportedLength: exportedVdis.length,
})
return
}
const deltaChainLength = Math.max(
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
)
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
/**
* For each snapshot VDI in `snapshotVdis`, maps it to its live source VDI,
* attaches its CONTENT_KEY when present, calls checkBaseVdis on all writers,
* and returns both the full mapping and the subset confirmed present by writers.
*
* @param {import('@vates/types').XenApiVdi[]} snapshotVdis - Snapshot VDI records to use as base candidates
* @param {Record<string, object>} srcVdisByRef - Live VM VDI records keyed by $ref
* @param {{ useContentKey?: boolean }} [options]
* @returns {Promise<{ presentBaseVdis: Map<string, string>, baseUuidToSrcVdiUuid: Map<string, string>}>}
*/
async _checkBaseVdis(snapshotVdis, srcVdisByRef, { useContentKey = false } = {}) {
const baseUuidToSrcVdiUuid = new Map()
const baseUuidToContentKey = new Map()
for (const snapshotVdi of snapshotVdis) {
const baseUuid = snapshotVdi.uuid
const srcVdi = srcVdisByRef[snapshotVdi.snapshot_of]
for (const lastExportedVdi of lastExportedVdis) {
const baseUuid = lastExportedVdi.uuid
const snapshotOf = lastExportedVdi.snapshot_of
const srcVdi = srcVdis[snapshotOf]
if (srcVdi !== undefined) {
baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
if (useContentKey) {
const contentKey = snapshotVdi.other_config[CONTENT_KEY]
if (contentKey !== undefined) {
baseUuidToContentKey.set(baseUuid, contentKey)
}
}
} else {
debug('ignore snapshot VDI because no longer present on VM', { vdi: baseUuid })
debug('ignore snapshot VDI because no longer present on VM', {
vdi: baseUuid,
})
}
}
const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
await this._callWriters(
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseUuidToContentKey),
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
'writer.checkBaseVdis()',
false
)
return { presentBaseVdis, baseUuidToSrcVdiUuid }
}
async _selectBaseVm() {
const xapi = this._xapi
const exportedVdis = this._jobSnapshotVdis.filter(
_ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
)
const grouped = this._groupSnapshotVdisByDatetime(exportedVdis)
this._baseVdis = {}
if (grouped.length === 0) {
debug('no base VDIs found', {
jobLength: this._jobSnapshotVdis.length,
exportedLength: exportedVdis.length,
})
}
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
let lastExportedVdis = []
let presentBaseVdis = new Map()
let baseUuidToSrcVdiUuid = new Map()
let deltaChainLength
if (grouped.length > 0) {
lastExportedVdis = grouped[0].vdis
// only compute it on the _jobSnapshotVdis , not on the reused snapshot from CONTENT_KEYS
deltaChainLength = Math.max(
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
)
;({ presentBaseVdis, baseUuidToSrcVdiUuid } = await this._checkBaseVdis(lastExportedVdis, srcVdis))
}
// if we don't find any candidates
// if there is only one writer of type IncrementalXapiWriter
// allow the reuse of any matching snapshot of this VM even from another job
if (
presentBaseVdis.size === 0 &&
this._writers.size === 1 &&
[...this._writers][0] instanceof IncrementalXapiWriter
) {
debug('fallback to content key')
const candidates = {}
for (const srcVdi of Object.values(srcVdis)) {
const snapshots = srcVdi.$snapshots
debug(`${srcVdi.uuid} got ${snapshots.length} snapshots`)
for (const snapshot of snapshots) {
if (DATETIME in snapshot.other_config) {
candidates[snapshot.uuid] = snapshot
}
}
}
debug('got candidates ', Object.keys(candidates).length)
this._filterValidSnapshotVdis(candidates)
debug('got filtered ', Object.keys(candidates).length)
for (const group of this._groupSnapshotVdisByDatetime(Object.values(candidates))) {
debug('check by datetime ', [...group.vdis].length)
debug(group.vdis)
const result = await this._checkBaseVdis(group.vdis, srcVdis, { useContentKey: true })
debug('result ', result)
if (result.presentBaseVdis.size > 0) {
lastExportedVdis = group.vdis
presentBaseVdis = result.presentBaseVdis
baseUuidToSrcVdiUuid = result.baseUuidToSrcVdiUuid
break
}
debug('try next group if any')
}
} else {
debug(' no fallback ')
}
if (presentBaseVdis.size === 0) {
debug('no base VM found')

View File

@@ -236,7 +236,28 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
// ensure they are attached to only one vm snapshot
// ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
_filterValidSnapshotVdis(vdiCandidates) {
async _fetchJobSnapshots() {
const jobId = this._jobId
const xapi = this._xapi
const vdiCandidates = {}
const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
Object.values(xapi.objects.indexes.type.VDI)
.filter(_ => !!_) // filter nullish
.filter(({ other_config, snapshot_of }) => {
return (
vdiUuids.includes(snapshot_of) &&
other_config[JOB_ID] === jobId &&
other_config[VM_UUID] === this._vm.uuid &&
other_config[COPY_OF] === undefined
)
})
.forEach(vdi => {
vdiCandidates[vdi.uuid] = vdi
})
// check that user snapshots are clean
for (const vdi of Object.values(vdiCandidates)) {
// cbt metadata are always considered linked to a backup job
// if they have the right other_config
@@ -258,7 +279,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
const userVms = vbds.map(({ $VM }) => $VM)
if (vbds.length > 1) {
warn(
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
This disk snapshot will be excluded from the backup cleaning`,
{ vdi, userVms }
)
@@ -273,7 +294,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
// => we exclude these from the backup processing
if (vm.$snapshot_of === undefined) {
warn(
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
This disk snapshot will be excluded from the backup cleaning`,
{ vdi, vm }
)
@@ -290,7 +311,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
) {
warn(
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
This disk snapshot will be excluded from the backup cleaning`,
{ vdi, vm }
)
@@ -307,7 +328,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
.forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
warn(
`vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
{ vdi, vm, vbds, outOfSnapshotsVdi }
)
@@ -315,47 +336,8 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
delete vdiCandidates[vdi.uuid]
})
}
}
async _fetchJobSnapshots() {
const jobId = this._jobId
const xapi = this._xapi
const vdiCandidates = {}
const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
Object.values(xapi.objects.indexes.type.VDI)
.filter(_ => !!_) // filter nullish
.filter(({ other_config, snapshot_of }) => {
return (
vdiUuids.includes(snapshot_of) &&
other_config[JOB_ID] === jobId &&
other_config[VM_UUID] === this._vm.uuid &&
other_config[COPY_OF] === undefined
)
})
.forEach(vdi => {
vdiCandidates[vdi.uuid] = vdi
})
this._filterValidSnapshotVdis(vdiCandidates)
this._jobSnapshotVdis = Object.values(vdiCandidates)
// For VMs with no disks, retention must be tracked directly on VM snapshots
// since there are no VDIs to anchor the discovery.
if (vdiUuids.length === 0) {
this._disklessJobSnapshotVms = this._vm.$snapshots
.filter(Boolean)
.filter(
({ other_config, $snapshot_of }) =>
$snapshot_of !== undefined &&
other_config[JOB_ID] === jobId &&
other_config[VM_UUID] === this._vm.uuid &&
other_config[COPY_OF] === undefined
)
} else {
this._disklessJobSnapshotVms = []
}
}
async _removeUnusedSnapshots() {
@@ -367,10 +349,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await xapi.barrier()
// ensure cached object are up to date
this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
// get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
// get the datetime of the most recent snapshot
const lastSnapshotDateTime = this._jobSnapshotVdis
.map(({ other_config }) => other_config[DATETIME])
.sort()
.pop()
@@ -447,28 +428,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
})
})
// Retention for VMs with no disks: VM snapshots are not reachable via VDIs
if (disklessVmSnapshots.length > 0) {
const snapshotsPerSchedule = groupBy(disklessVmSnapshots, _ => _.other_config[SCHEDULE_ID])
await asyncEach(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
// we only have one snapshot per date time since it's at the VM level
const snapshotPerDatetime = Object.fromEntries(snapshots.map(s => [s.other_config[DATETIME], s.$ref]))
const datetimes = Object.keys(snapshotPerDatetime).sort()
const settings = {
...baseSettings,
...allSettings[scheduleId],
...allSettings[this._vm.uuid],
}
const retention = settings.snapshotRetention ?? 0
await asyncEach(getOldEntries(retention, datetimes), async datetime => {
if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
return
}
await xapi.VM_destroy(snapshotPerDatetime[datetime])
})
})
}
// list and remove the snapshot were the jobs failed between
// makesnapshot and update_other_config
const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)

View File

@@ -229,13 +229,14 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
Object.entries(deltaExport.disks),
async ([diskRef, disk]) => {
const path = `${this._vmBackupDir}/${vhds[diskRef]}`
size += await adapter.writeVhd(path, disk, {
await adapter.writeVhd(path, disk, {
// no checksum for VHDs, because they will be invalidated by
// merges and chains
checksum: false,
validator: tmpPath => checkVhd(handler, tmpPath),
writeBlockConcurrency: this._config.writeBlockConcurrency,
})
size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
},
{
concurrency: settings.diskPerVmConcurrency,

View File

@@ -19,14 +19,11 @@ import {
REPLICATED_TO_SR_UUID,
DATETIME,
VM_UUID,
CONTENT_KEY,
resetVmOtherConfig,
} from '../../_otherConfig.mjs'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { XapiDiskSource } from '@xen-orchestra/xapi'
import { asyncEach } from '@vates/async-each'
import { createLogger } from '@xen-orchestra/log'
import { VM_POWER_STATE } from '@vates/types'
const { debug } = createLogger('xo:backups:IncrementalXapiWriter')
@@ -35,20 +32,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
// Built by checkBaseVdis, consumed by #decorateVmMetadata to set baseVdi.
#baseVdisBySourceUuid = new Map()
/**
* Finds VDIs on the target SR that can serve as base for the next delta transfer.
*
* For each entry in `baseUuidToSrcVdi`, searches the target SR for a matching snapshot
* using CONTENT_KEY when available, then falls back to COPY_OF matching for older snapshots.
* Entries with no matching base found on the target SR are removed from `baseUuidToSrcVdi`.
*
* Side-effects: populates `#baseVdisBySourceUuid`; may set `_targetVmRef`.
*
* @param {Map<string, string>} baseUuidToSrcVdi - Source snapshot UUID → source active VDI UUID. Mutated in place.
* @param {Map<string, string>} contentKeys - Source snapshot UUID → CONTENT_KEY value (empty for pre-CONTENT_KEY snapshots).
* @returns {Promise<void>}
*/
async checkBaseVdis(baseUuidToSrcVdi, contentKeys) {
async checkBaseVdis(baseUuidToSrcVdi) {
const sr = this._sr
this.#baseVdisBySourceUuid = new Map()
@@ -61,45 +45,18 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
// look for the same snapshot
// ensure there are no data between the snapshot and the active disk
const snapshotCandidates = new Map()
const snapshotCandidates = sr.$VDIs.filter(vdi => {
return (
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[JOB_ID] === this._job.id &&
vdi.other_config[VM_UUID] === this._vmUuid &&
baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
)
})
debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.length)
for (const [baseUuid, srcVdiuid] of baseUuidToSrcVdi) {
let target
const contentKey = contentKeys.get(baseUuid)
if (contentKey !== undefined) {
debug('got one content key, look for a candidate')
target = sr.$VDIs.find(
vdi =>
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[CONTENT_KEY] === contentKey && // &&
// ensure we don't replicate on ourself or any of the vdi in the source chain
vdi.$snapshot_of.uuid !== srcVdiuid
)
}
// fall back for older snapshots
if (target === undefined) {
debug('content key not here or not found , look by jobid ')
target = sr.$VDIs.find(
vdi =>
vdi?.managed &&
vdi?.is_a_snapshot &&
vdi.other_config[JOB_ID] === this._job.id &&
vdi.other_config[VM_UUID] === this._vmUuid &&
vdi?.other_config[COPY_OF] === baseUuid
)
}
if (target !== undefined) {
snapshotCandidates.set(baseUuid, target)
}
}
debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.size)
if (snapshotCandidates.size > 0) {
if (snapshotCandidates.length > 0) {
// reset before searching for candidates
this.#baseVdisBySourceUuid = new Map()
this._targetVmRef = undefined
@@ -107,12 +64,10 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
for (const [sourceUuid, vdi] of baseVdisBySourceUuid) {
this.#baseVdisBySourceUuid.set(sourceUuid, vdi)
}
debug(' this.#baseVdisBySourceUuid ', this.#baseVdisBySourceUuid.size)
if (targetVmRef !== undefined) {
this._targetVmRef = targetVmRef
}
} else {
debug('legacy fallback ( no content key ) ')
// Legacy fallback (upgrade from pre-6.3): no target snapshots exist yet,
// look for active (non-snapshot) VDIs with matching COPY_OF, like the old code did.
debug('checkBaseVdis, no snapshot candidates, falling back to legacy active VDI lookup')
@@ -137,9 +92,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
* 6.3+ snapshot-based validation: for each snapshot candidate, check whether
* the active VDI has diverged from the snapshot. Returns a baseVdisBySourceUuid
* map and, when all disks are clean, the targetVmRef to reuse.
*
* @param {Map<XenApiVdi['id'], import('@vates/types').XenApiVdi>} snapshotCandidates - Snapshot VDIs on the target SR to validate.
* @returns {Promise<{ baseVdisBySourceUuid: Map<string, import('@vates/types').XenApiVdi>, targetVmRef: import('@vates/types').XenApiVm['$ref'] | undefined }>}
*/
async #validateSnapshotCandidates(snapshotCandidates) {
const sr = this._sr
@@ -148,36 +100,24 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
let canChainToTargetVm = true
await asyncEach(
snapshotCandidates.entries(),
async ([sourceUuid, snapshot]) => {
snapshotCandidates,
async snapshot => {
let diffDisk
let activeVdi
try {
activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
if (userVbds.length !== 1) {
canChainToTargetVm = false
debug('checkBaseVdis, shared vbd ', { ref: snapshot.$ref, userVbds })
// shared vdi ignore / don't chain
// shared vdi ignore
return
}
const vm = userVbds[0].$VM
// a running VM will fail to compute disk exports
// also a running VM can be assumed to have changed data
if (vm.power_state !== VM_POWER_STATE.HALTED && vm.power_state !== VM_POWER_STATE.SUSPENDED) {
canChainToTargetVm = false
debug('checkBaseVdis, target vm is not halted or suspended', {
ref: snapshot.$ref,
userVbds,
powerState: vm.power_state,
})
}
// from this disk of from another
// skip the costly part, only do the disk chaining
if (!canChainToTargetVm) {
debug("Can't chain VM anyway , fast return and chain with snapshot")
baseVdisBySourceUuid.set(sourceUuid, snapshot)
if (!('start' in vm.blocked_operations)) {
debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
// vm start unlocked
// not really an issue since we have check the delta
// but it indicates the users played with the blocked operations
return
}
diffDisk = new XapiDiskSource({
@@ -187,16 +127,18 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
onlyListChangedBlocks: true,
})
await diffDisk.init()
const sourceUuid = snapshot.other_config?.[COPY_OF]
if (diffDisk.getBlockIndexes().length === 0) {
debug(' NO CHANGE , source detected ? ', !!sourceUuid)
// no block modification since the common snapshot, we can chain VM and disk
// the disk is chained with the active to keep the chain linear
baseVdisBySourceUuid.set(sourceUuid, activeVdi)
if (sourceUuid) {
baseVdisBySourceUuid.set(sourceUuid, activeVdi)
}
// Track the target VM (the replicated VM to update on the next transfer).
targetVmRef = vm.$ref
} else {
debug(' GOT CHANGE, source detected ? ', !!sourceUuid)
baseVdisBySourceUuid.set(sourceUuid, snapshot)
if (sourceUuid) {
baseVdisBySourceUuid.set(sourceUuid, snapshot)
}
// there are changed block since the snapshot
// we can reuse it to transfer a delta, but we will
// create a new VM
@@ -223,7 +165,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
)
if (!canChainToTargetVm) {
debug('checkBaseVdis,NOT a valid vm target')
// if at least one disk has new data, create a new VM
// instead of updating it
targetVmRef = undefined
@@ -231,7 +172,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
debug('checkBaseVdis,got a valid vm target', targetVmRef)
}
debug('checkBaseVdis,base vdis found : ', baseVdisBySourceUuid.size)
return { baseVdisBySourceUuid, targetVmRef }
}
@@ -329,10 +269,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
}
Object.values(backup.vdis).forEach(vdi => {
// setVmSnapshotContentKeys sets CONTENT_KEY = the snapshot VDI's own UUID, but
// that XenAPI write may not be visible in the xapi cache yet when exportIncrementalVm
// reads vdi.other_config. Derive the correct value directly instead of relying on cache.
vdi.other_config[CONTENT_KEY] = vdi.uuid
vdi.other_config[COPY_OF] = vdi.uuid
vdi.other_config[JOB_ID] = job.id
vdi.other_config[SCHEDULE_ID] = scheduleId
@@ -391,7 +327,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
await xapi.VM_snapshot(targetVmRef, {
name_label: `${vm.name_label} - ${job.name} / ${schedule.name} ${formatFilenameDate(timestamp)}`,
})
await resetVmOtherConfig(xapi, targetVmRef)
})
return {

View File

@@ -31,6 +31,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
return await Task.run({ properties: { name: 'clean-vm' } }, () => {
return this._adapter.cleanVm(this._vmBackupDir, {
...options,
fixMetadata: true,
logInfo: info,
logWarn: (message, data) => {
warn(message, data)
@@ -38,6 +39,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
},
lock: false,
mergeBlockConcurrency: this._config.mergeBlockConcurrency,
removeTmp: true,
})
})
} catch (error) {

View File

@@ -2,7 +2,7 @@
/**
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
*/
import assert from 'assert'
@@ -10,15 +10,15 @@ import { createLogger } from '@xen-orchestra/log'
import { basename, dirname } from 'path'
import { asyncEach } from '@vates/async-each'
import { relativeFromFile, normalize, resolveFromFile } from '@xen-orchestra/fs/path'
import { relativeFromFile } from '@xen-orchestra/fs/path'
// @ts-ignore
const { warn } = createLogger('remote-disk:merge')
/**
* @typedef {Object} MergeState
* @property {{ uuid?: string }} child
* @property {{ uuid?: string }} parent
* @property {{ uuid: string }} child
* @property {{ uuid: string }} parent
* @property { string[] | undefined} chain
* @property {number} currentBlock
* @property {number} mergedDataSize
@@ -82,12 +82,12 @@ export class MergeRemoteDisk {
#lastStateWrittenAt
/**
* @type {RemoteHandlerAbstract}
* @type {FileAccessor}
*/
#handler
/**
* @param {RemoteHandlerAbstract} handler
* @param {FileAccessor} handler
* @param {Object} params
* @param {Function} [params.onProgress]
* @param {Logger | Function} [params.logInfo]
@@ -222,7 +222,6 @@ export class MergeRemoteDisk {
await this.#mergeBlocks(parentDisk, childDisk)
await parentDisk.flushMetadata(childDisk)
await parentDisk.mergeMetadata(childDisk)
this.#state.diskSize = parentDisk.getSizeOnDisk()
}
/**
@@ -267,6 +266,8 @@ export class MergeRemoteDisk {
await this.#writeState()
this.#state.diskSize = childDisk.getSizeOnDisk()
this.#onProgress({ total: nBlocks, done: nBlocks })
}
@@ -321,44 +322,4 @@ export class MergeRemoteDisk {
return { mergedDataSize, finalDiskSize }
}
/**
* Scans vdiDir files for interrupted merge state files and resolves their chains
*
* @param {RemoteHandlerAbstract} handler
* @param {string} vdiDir
* @param {string[]} files - all file paths in vdiDir (prependDir: true)
* @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
* @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
*/
static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
const result = new Map()
for (const filePath of files) {
const match = STATE_FILE_RE.exec(basename(filePath))
if (match === null) continue
const parentPath = normalize(vdiDir + '/' + match[1])
let chain
try {
const state = JSON.parse(await handler.readFile(filePath))
if (Array.isArray(state?.chain)) {
chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
} else {
// old state file without chain: fall back to UUID lookup then header-based map
const childUuid = state?.child?.uuid
const childPath = childUuid !== undefined ? uuidToPath.get(childUuid) : childOf.get(parentPath)
if (childPath !== undefined) chain = [parentPath, childPath]
}
} catch (error) {
warn("Merge state unreadable, can't restart merging.", error)
}
result.set(parentPath, { stateFilePath: filePath, chain })
}
return result
}
}

View File

@@ -73,30 +73,6 @@ export class RemoteDisk extends RandomAccessDisk {
throw new Error(`getUuid must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getParentUuid() {
throw new Error(`getParentUuid must be implemented`)
}
/**
* Abstract
* @returns {string}
*/
getParentPath() {
throw new Error(`getParentPath must be implemented`)
}
/**
* Abstract
* @returns {object}
*/
getMetadata() {
throw new Error(`getMetadata must be implemented`)
}
/**
* Abstract
* @returns {Promise<boolean>} canMergeConcurently
@@ -112,18 +88,6 @@ export class RemoteDisk extends RandomAccessDisk {
return Math.ceil(this.getVirtualSize() / this.getBlockSize())
}
/**
* Returns true if every block present in `other` is also present in this disk.
* @param {RemoteDisk} other
* @returns {boolean}
*/
containsAllDataOf(other) {
for (const blockIndex of other.getBlockIndexes()) {
if (!this.hasBlock(blockIndex)) return false
}
return true
}
/**
* Checks if the VHD contains a specific block.
* @param {number} index
@@ -241,34 +205,11 @@ export class RemoteDisk extends RandomAccessDisk {
/**
* Abstract
* Deletes alias/disk/disks
* @param {Object} options
*/
async unlink({ force = false } = {}) {
async unlink() {
throw new Error(`unlink must be implemented`)
}
/**
* Checks the integrity of the disk's external reference (e.g. alias target).
* @param {Object} [opts]
* @param {boolean} [opts.remove]
* @param {Function} [opts.logWarn]
* @param {Function} [opts.logInfo]
* @returns {Promise<string | undefined>}
*/
async clean(opts = {}) {
return undefined
}
/**
* Returns all file paths within dir that this disk claims.
*
* @param {string} dir
* @returns {Promise<string[]>}
*/
async listAssociatedFiles(dir) {
throw new Error(`listAssociatedFiles must be implemented`)
}
/**
* @returns {Promise<RemoteDisk>}
*/

View File

@@ -3,9 +3,9 @@
/**
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
* @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
* @typedef {import('vhd-lib/_createFooterHeader').VhdFooter} VhdFooter
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
*
*/
@@ -17,7 +17,6 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
import { stringify } from 'uuid'
import { dirname, join } from 'node:path'
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
import { normalize } from '@xen-orchestra/fs/path'
export class RemoteVhdDisk extends RemoteDisk {
/**
@@ -26,7 +25,7 @@ export class RemoteVhdDisk extends RemoteDisk {
#path
/**
* @type {RemoteHandlerAbstract}
* @type {FileAccessor}
*/
#handler
@@ -57,7 +56,7 @@ export class RemoteVhdDisk extends RemoteDisk {
/**
* @param {Object} params
* @param {RemoteHandlerAbstract} params.handler
* @param {FileAccessor} params.handler
* @param {string} params.path
*/
constructor({ handler, path }) {
@@ -70,7 +69,6 @@ export class RemoteVhdDisk extends RemoteDisk {
/**
* @param {Object} [options]
* @param {boolean} [options.force=false]
* @param {boolean} [options.ignoreBlockIndexes=false]
* @returns {Promise<void>}
*/
async init(options = {}) {
@@ -87,9 +85,7 @@ export class RemoteVhdDisk extends RemoteDisk {
}
this.#dispose = dispose
if (!options.ignoreBlockIndexes) {
await this.#vhd.readBlockAllocationTable()
}
await this.#vhd.readBlockAllocationTable()
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
} catch (error) {
await this.close()
@@ -176,20 +172,6 @@ export class RemoteVhdDisk extends RemoteDisk {
return stringify(this.#vhd.header.parentUuid)
}
/**
* Returns the parent path
*
* @returns {string}
*/
getParentPath() {
if (this.#vhd === undefined) {
throw new Error(`can't call getParentPath of a RemoteVhdDisk before init`)
}
const parentPath = this.#vhd.header.parentUnicodeName
return normalize(join(dirname(this.#path), parentPath))
}
/**
* @returns {Promise<boolean>} canMergeConcurently
*/
@@ -236,7 +218,7 @@ export class RemoteVhdDisk extends RemoteDisk {
}
const parentPath = this.#vhd.header.parentUnicodeName
const fullParentPath = normalize(join(dirname(this.#path), parentPath))
const fullParentPath = join(dirname(this.#path), parentPath)
if (!parentPath) {
throw new Error(`disk ${this.#path} doesn't have parents`)
@@ -245,13 +227,6 @@ export class RemoteVhdDisk extends RemoteDisk {
const parent = new RemoteVhdDisk({ handler: this.#handler, path: fullParentPath })
return parent
}
/**
* Parent of a RemoteVhdDisk is a RemoteVhdDisk
* @returns {Promise<RemoteVhdDisk>}
*/
async openParent() {
return /** @type {Promise<RemoteVhdDisk>} */ (super.openParent())
}
/**
* Writes a full block into this VHD.
@@ -398,7 +373,7 @@ export class RemoteVhdDisk extends RemoteDisk {
}
/**
* @param {RemoteDisk} childDisk
* @param {RemoteVhdDisk} childDisk
* @returns {Promise<void>}
*/
async mergeMetadata(childDisk) {
@@ -441,9 +416,10 @@ export class RemoteVhdDisk extends RemoteDisk {
if (isVhdAlias(newPath)) {
const dataPath = await resolveVhdAlias(this.#handler, this.#path)
await this.#handler.unlink(newPath)
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
await this.#handler.unlink(this.#path)
await this.#handler.unlink(newPath)
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
this.#path = newPath
} else {
@@ -463,59 +439,31 @@ export class RemoteVhdDisk extends RemoteDisk {
/**
* Deletes disk
* @param {Object} options
*/
async unlink({ force = false } = {}) {
async unlink() {
if (this.#vhd === undefined) {
if (force) {
let resolved = this.#path
try {
resolved = await resolveVhdAlias(this.#handler, this.#path)
} catch (err) {
// broken vhd directory must be unlinkable
if (err.code !== 'EISDIR') {
throw err
}
// warn('Deleting directly a VhdDirectory', { this.#path, err })
}
try {
await this.#handler.unlink(resolved)
} catch (err) {
if (err.code === 'EISDIR') {
await this.#handler.rmtree(resolved)
} else {
throw err
}
}
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
}
// also delete the alias file
if (this.#path !== resolved) {
await this.#handler.unlink(this.#path)
}
} else {
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
}
} else {
await this.close()
if (isVhdAlias(this.#path)) {
try {
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
} catch (err) {
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
}
}
}
await this.close()
if (isVhdAlias(this.#path)) {
try {
await this.#handler.unlink(this.#path)
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
} catch (err) {
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
await this.#handler.rmtree(this.#path).catch(() => {})
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
}
}
}
try {
await this.#handler.unlink(this.#path)
} catch (err) {
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
await this.#handler.rmtree(this.#path).catch(() => {})
}
}
}
/**
@@ -529,118 +477,4 @@ export class RemoteVhdDisk extends RemoteDisk {
return this.#vhd instanceof VhdDirectory
}
/**
* Returns all file paths within dir that this disk claims.
* For a plain VHD or VHD directory: [path]
* For a VHD alias: [aliasPath, resolvedDataPath]
* If the alias target cannot be resolved, only the alias path is returned.
*
* @param {string} dir
* @returns {Promise<string[]>}
*/
async listAssociatedFiles(dir) {
const prefix = normalize(dir.endsWith('/') ? dir : dir + '/')
const isInDir = /** @param {string} p */ p => p === dir || p.startsWith(prefix)
const files = []
if (isInDir(this.#path)) {
files.push(this.#path)
}
if (isVhdAlias(this.#path)) {
try {
const resolved = await resolveVhdAlias(this.#handler, this.#path)
if (isInDir(resolved)) {
files.push(resolved)
}
} catch {
// broken alias, no data file to claim
}
}
return files
}
/**
* Checks the integrity of this disk's alias reference.
* Only meaningful for alias files (.alias.vhd); no-op for plain VHDs.
* Returns the resolved target path when the alias is valid, undefined otherwise.
*
* @param {Object} [opts]
* @param {boolean} [opts.remove]
* @param {Function} [opts.logWarn]
* @param {Function} [opts.logInfo]
* @returns {Promise<string | undefined>}
*/
async clean({ remove = false, logWarn = () => {}, logInfo = () => {} } = {}) {
if (!isVhdAlias(this.#path)) {
return undefined
}
let target
try {
target = await resolveVhdAlias(this.#handler, this.#path)
} catch (err) {
if (err.code === 'ENOENT') {
logWarn('missing target of alias', { alias: this.#path })
if (remove) {
logInfo('removing alias with missing target', { alias: this.#path })
await this.#handler.unlink(this.#path)
}
return undefined
}
if (err.code === 'EISDIR') {
logWarn('alias is a vhd directory', { alias: this.#path })
if (remove) {
logInfo('removing vhd directory named as alias', { alias: this.#path })
await VhdAbstract.unlink(this.#handler, this.#path)
}
return undefined
}
logWarn('unhandled error while checking alias', { alias: this.#path, err })
return undefined
}
if (target === '') {
logWarn('empty target for alias', { alias: this.#path })
if (remove) {
logInfo('removing alias with empty target', { alias: this.#path })
await this.#handler.unlink(this.#path)
}
return undefined
}
if (!isVhdAlias(target) && !target.endsWith('.vhd')) {
logWarn('alias references non VHD target', { alias: this.#path, target })
if (remove) {
logInfo('removing alias and non VHD target', { alias: this.#path, target })
await this.#handler.unlink(target)
await this.#handler.unlink(this.#path)
}
return undefined
}
try {
const { dispose } = await openVhd(this.#handler, target)
try {
await dispose()
} catch (_) {
// errors during dispose should not trigger deletion
}
return target
} catch (error) {
logWarn('missing or broken alias target', { alias: this.#path, target, error })
if (remove) {
try {
await VhdAbstract.unlink(this.#handler, this.#path)
} catch (/** @type {any} */ err) {
if (err.code !== 'ENOENT') {
logWarn('error deleting broken alias', { alias: this.#path, target, err })
}
}
}
return undefined
}
}
}

View File

@@ -4,6 +4,7 @@
* @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
* @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
*/
import { RemoteDisk } from './RemoteDisk.mjs'
@@ -36,6 +37,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
/**
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {RemoteVhdDisk[]} params.disks
*/
constructor({ disks }) {
@@ -246,7 +248,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
}
/**
* @param {RemoteDisk} childDisk
* @param {RemoteVhdDisk} childDisk
* @returns {Promise<void>}
*/
async flushMetadata(childDisk) {
@@ -254,7 +256,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
}
/**
* @param {RemoteDisk} childDisk
* @param {RemoteVhdDisk} childDisk
* @returns {Promise<void>}
*/
mergeMetadata(childDisk) {
@@ -280,11 +282,10 @@ export class RemoteVhdDiskChain extends RemoteDisk {
/**
* Deletes all the disks
* @param {Object} options
*/
async unlink({ force = false } = {}) {
async unlink() {
for (const disk of this.#disks) {
await disk.unlink({ force })
await disk.unlink()
}
}
@@ -300,16 +301,4 @@ export class RemoteVhdDiskChain extends RemoteDisk {
}
return true
}
/**
* Checks the integrity of the disk's external reference (e.g. alias target).
* @param {Object} [opts]
* @param {boolean} [opts.remove]
* @param {Function} [opts.logWarn]
* @param {Function} [opts.logInfo]
* @returns {Promise<string | undefined>}
*/
async clean(opts = {}) {
throw new Error(`Can't clean a disk chain`)
}
}

View File

@@ -0,0 +1,49 @@
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
export { RemoteDisk } from './RemoteDisk.mjs'
export { openDiskChain } from './openDiskChain.mjs'
/**
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
*/
/**
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {string} params.path
* @returns {Promise<RemoteDisk>}
*/
export async function openDisk({ handler, path }) {
const disk = new RemoteVhdDisk({ handler, path })
await disk.init()
return disk
}
/**
*
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {string} params.path
* @returns {Promise<Disposable<RemoteDisk>>}
*/
export async function openDisposableDisk({ handler, path }) {
const disk = new RemoteVhdDisk({ handler, path })
await disk.init()
return {
value: disk,
dispose: () => disk.close(),
}
}
const DISK_EXTENSIONS = ['.vhd']
/**
* Returns true if the path points to a supported disk format.
*
* @param {FileAccessor} _handler - Remote file handler (reserved for future use)
* @param {string} path - Path to check
* @returns {boolean}
*/
export function isDisk(_handler, path) {
return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
}

View File

@@ -0,0 +1,40 @@
// @ts-check
/**
*
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
*/
import { DiskChain } from '@xen-orchestra/disk-transform'
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
import { defer } from 'golike-defer'
/**
* @param {any} $defer
* @param {Object} params
* @param {FileAccessor} params.handler
* @param {string} params.path
* @param {string | undefined} params.until
*/
async function _openDiskChain($defer, { handler, path, until }) {
let disk
/**
* @type {Array<RemoteDisk>}
*/
const disks = []
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
disk = new RemoteVhdDisk({ handler, path })
await disk.init()
disks.push(disk)
while (disk.isDifferencing()) {
disk = await disk.openParent()
if (disk.getPath() === until) {
break
}
disks.unshift(disk)
}
// the root disk
return new DiskChain({ disks })
}
export const openDiskChain = defer(_openDiskChain)

View File

@@ -8,7 +8,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.72.1",
"version": "0.72.0",
"engines": {
"node": ">=14.18"
},
@@ -30,14 +30,12 @@
"@vates/nbd-client": "^3.4.0",
"@vates/parse-duration": "^0.1.1",
"@vates/task": "^0.7.0",
"@vates/types": "^1.24.0",
"@xen-orchestra/async-map": "^0.1.3",
"@xen-orchestra/disk-transform": "^1.2.3",
"@xen-orchestra/fs": "^4.8.0",
"@xen-orchestra/log": "^0.7.2",
"@xen-orchestra/qcow2": "^1.3.0",
"@xen-orchestra/template": "^0.1.1",
"@xen-orchestra/backup-archive": "^1.0.0",
"app-conf": "^3.0.0",
"compare-versions": "^6.0.0",
"d3-time-format": "^4.1.0",
@@ -63,8 +61,7 @@
"fs-extra": "^11.1.0",
"rimraf": "^6.0.1",
"sinon": "^18.0.0",
"tmp": "^0.2.1",
"typescript": "^5.9.3"
"tmp": "^0.2.1"
},
"peerDependencies": {
"@xen-orchestra/xapi": "^8.7.2"
@@ -75,6 +72,7 @@
"url": "https://vates.fr"
},
"exports": {
"./disks": "./disks/index.mjs",
"./*": "./*"
}
}

View File

@@ -1,47 +0,0 @@
export namespace VHDFOOTER {
let cookie: string
let features: number
let fileFormatVersion: number
let dataOffset: number
let timestamp: number
let creatorApplication: string
let creatorVersion: number
let creatorHostOs: number
let originalSize: number
let currentSize: number
namespace diskGeometry {
let cylinders: number
let heads: number
let sectorsPerTrackCylinder: number
}
let diskType: number
let checksum: number
let uuid: Buffer<ArrayBuffer>
let saved: string
let hidden: string
let reserved: string
}
export namespace VHDHEADER {
let cookie_1: string
export { cookie_1 as cookie }
let dataOffset_1: any
export { dataOffset_1 as dataOffset }
export let tableOffset: number
export let headerVersion: number
export let maxTableEntries: number
export let blockSize: number
let checksum_1: number
export { checksum_1 as checksum }
export let parentUuid: any
export let parentTimestamp: number
export let reserved1: number
export let parentUnicodeName: string
export let parentLocatorEntry: {
platformCode: number
platformDataSpace: number
platformDataLength: number
reserved: number
platformDataOffset: number
}[]
export let reserved2: string
}

View File

@@ -20,8 +20,7 @@
},
"dependencies": {
"@vates/async-each": "^1.0.3",
"@xen-orchestra/backups": "^0.72.1",
"@xen-orchestra/backup-archive": "^1.0.0",
"@xen-orchestra/backups": "^0.72.0",
"@xen-orchestra/disk-transform": "^1.2.3",
"@xen-orchestra/fs": "^4.8.0",
"@xen-orchestra/qcow2": "^1.3.0",

View File

@@ -1,11 +1,11 @@
import Disposable from 'promise-toolbox/Disposable'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { openDisposableDisk, RemoteDisk } from '@xen-orchestra/backup-archive/disks'
import { openDisposableDisk } from '@xen-orchestra/backups/disks'
import { formatBytes } from '../utils.mjs'
export async function infoCommand(handlerUrl: string, diskPath: string, _extraArgs: string[]): Promise<void> {
await Disposable.use(getSyncedHandler({ url: handlerUrl }), async handler => {
await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async (disk: RemoteDisk) => {
await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async disk => {
const isDifferencing = disk.isDifferencing()
const virtualSize = disk.getVirtualSize()
const blockSize = disk.getBlockSize()

View File

@@ -1,6 +1,6 @@
import Disposable from 'promise-toolbox/Disposable'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { isDisk, openDisposableDisk, RemoteDisk } from '@xen-orchestra/backup-archive/disks'
import { isDisk, openDisposableDisk } from '@xen-orchestra/backups/disks'
import { formatBytes, renderTable } from '../utils.mjs'
import { asyncEach } from '@vates/async-each'
const HEADERS = ['File', 'UID', 'Size on disk', 'Virtual size', 'Differencing', 'Parent UID']
@@ -87,21 +87,18 @@ export async function listCommand(handlerUrl: string, dirPath: string, _extraArg
async (diskPath: string) => {
const filename = diskPath.slice(diskPath.lastIndexOf('/') + 1)
try {
const disk = await Disposable.use(
openDisposableDisk({ handler, path: diskPath }),
async (disk: RemoteDisk) => {
const differencing = disk.isDifferencing()
return {
ok: true as const,
filename,
uid: disk.getUuid(),
sizeOnDisk: formatBytes(disk.getSizeOnDisk()),
virtualSize: formatBytes(disk.getVirtualSize()),
differencing,
parentUid: differencing ? disk.getParentUuid() : null,
}
const disk = await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async disk => {
const differencing = disk.isDifferencing()
return {
ok: true as const,
filename,
uid: disk.getUuid(),
sizeOnDisk: formatBytes(disk.getSizeOnDisk()),
virtualSize: formatBytes(disk.getVirtualSize()),
differencing,
parentUid: differencing ? disk.getParentUuid() : null,
}
)
})
disks.push(disk)
} catch (err) {
disks.push({ ok: false as const, filename, error: err instanceof Error ? err.message : String(err) })

View File

@@ -3,7 +3,7 @@ import { Readable } from 'node:stream'
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'
import Disposable from 'promise-toolbox/Disposable'
import { getSyncedHandler, type RemoteHandler } from '@xen-orchestra/fs'
import { openDisposableDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
import { openDisposableDisk, openDiskChain } from '@xen-orchestra/backups/disks'
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
import { toQcow2Stream } from '@xen-orchestra/qcow2'

View File

@@ -34,7 +34,7 @@ declare module 'vhd-lib/disk-consumer/index.mjs' {
export function toVhdStream(disk: RandomAccessDisk): Promise<Readable>
}
declare module '@xen-orchestra/backup-archive/disks/openDiskChain.mjs' {
declare module '@xen-orchestra/backups/disks/openDiskChain.mjs' {
import type { RemoteHandler } from '@xen-orchestra/fs'
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'
@@ -45,7 +45,7 @@ declare module '@xen-orchestra/backup-archive/disks/openDiskChain.mjs' {
}): Promise<RandomAccessDisk>
}
declare module '@xen-orchestra/backup-archive/disks' {
declare module '@xen-orchestra/backups/disks' {
import type { RemoteHandler } from '@xen-orchestra/fs'
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'

Some files were not shown because too many files have changed in this diff Show More