mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-15 19:42:26 -06:00
Compare commits
7 Commits
pbt_aggreg
...
web-core/r
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abc3018b66 | ||
|
|
423f00fefa | ||
|
|
7456ab9faf | ||
|
|
4d30d43a66 | ||
|
|
d1f3efc57f | ||
|
|
1d881fe8cc | ||
|
|
7a9de0de57 |
@@ -281,7 +281,7 @@ module.exports = {
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['@xen-orchestra/{web-core,lite,web}/src/route-map.d.ts'],
|
||||
files: ['@xen-orchestra/{web-core,lite,web}/typed-router.d.ts'],
|
||||
parserOptions: {
|
||||
sourceType: 'module',
|
||||
},
|
||||
|
||||
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@@ -25,7 +25,7 @@ jobs:
|
||||
- name: Setup Node environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '>=22.12'
|
||||
node-version: '24.14'
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Install project dependencies
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@vates/types",
|
||||
"private": false,
|
||||
"version": "1.24.0",
|
||||
"version": "1.23.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": {
|
||||
|
||||
@@ -3,5 +3,4 @@ export * from './xen-api.mjs'
|
||||
export * from './xo.mjs'
|
||||
export * from './xo-app.mjs'
|
||||
export * from './lib/xen-orchestra-xapi.mjs'
|
||||
export * from './lib/xen-orchestra-acl.mjs'
|
||||
export * from './lib/complex-matcher.mjs'
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
import { Branded } from '../common.mjs'
|
||||
import type { XoGroup, XoUser } from '../xo.mjs'
|
||||
|
||||
export type XoAclRole =
|
||||
| {
|
||||
id: Branded<'acl-v2-role'>
|
||||
name: string
|
||||
description?: string
|
||||
}
|
||||
| {
|
||||
id: Branded<'acl-v2-role'>
|
||||
name: string
|
||||
description?: string
|
||||
isTemplate: true
|
||||
roleTemplateId: number
|
||||
}
|
||||
|
||||
export type XoAclSupportedActionsByResource = {
|
||||
[resource: string]: Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* E.g
|
||||
* vms {
|
||||
* shutdown: {
|
||||
* clean: true,
|
||||
* hard: true,
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* `shutdown | shutdown:clean | shutdown:hard`
|
||||
*/
|
||||
export type GetKeysRecursively<T, Prefix extends string = ''> = {
|
||||
[K in keyof T]: T[K] extends object
|
||||
? K extends string
|
||||
? `${Prefix}${K}` | GetKeysRecursively<T[K], `${Prefix}${K}:`>
|
||||
: never
|
||||
: K extends string
|
||||
? `${Prefix}${K}`
|
||||
: never
|
||||
}[keyof T]
|
||||
|
||||
export type XoAclSupportedResource<TActionsByResource extends XoAclSupportedActionsByResource> =
|
||||
keyof TActionsByResource
|
||||
|
||||
export type XoAclSupportedActions<
|
||||
TActionsByResource extends XoAclSupportedActionsByResource,
|
||||
TResource extends XoAclSupportedResource<TActionsByResource>,
|
||||
> = (GetKeysRecursively<TActionsByResource[TResource]> & string) | '*'
|
||||
|
||||
export type XoAclBasePrivilege = {
|
||||
id: Branded<'acl-v2-privilege'>
|
||||
resource: string
|
||||
action: string
|
||||
selector?: string
|
||||
effect: 'allow' | 'deny'
|
||||
roleId: XoAclRole['id']
|
||||
}
|
||||
|
||||
export type XoAclPrivilege<
|
||||
TActionsByResource extends XoAclSupportedActionsByResource,
|
||||
TResource extends XoAclSupportedResource<TActionsByResource>,
|
||||
> = XoAclBasePrivilege & {
|
||||
resource: TResource
|
||||
action: XoAclSupportedActions<TActionsByResource, TResource>
|
||||
}
|
||||
|
||||
export type XoUserRole = {
|
||||
id: string
|
||||
roleId: XoAclRole['id']
|
||||
userId: XoUser['id']
|
||||
}
|
||||
|
||||
export type XoGroupRole = {
|
||||
id: string
|
||||
roleId: XoAclRole['id']
|
||||
groupId: XoGroup['id']
|
||||
}
|
||||
@@ -25,7 +25,6 @@ import type {
|
||||
} from '../common.mjs'
|
||||
import type { PassThrough, Readable } from 'node:stream'
|
||||
import type {
|
||||
XapiXoRecord,
|
||||
XoGpuGroup,
|
||||
XoHost,
|
||||
XoNetwork,
|
||||
@@ -37,7 +36,6 @@ import type {
|
||||
XoVm,
|
||||
XoVmTemplate,
|
||||
XoVif,
|
||||
XoVmSnapshot,
|
||||
} from '../xo.mjs'
|
||||
|
||||
export type XcpPatches = {
|
||||
@@ -74,19 +72,12 @@ export interface Xapi {
|
||||
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
|
||||
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
|
||||
connectVif(vifId: XoVif['id']): Promise<void>
|
||||
getField<T extends XenApiRecord, K extends keyof T>(
|
||||
type: Extract<WrappedXenApiRecord, T>['$type'],
|
||||
ref: T['$ref'],
|
||||
field: K
|
||||
): Promise<T[K]>
|
||||
getObject: <
|
||||
XoRecord extends XapiXoRecord,
|
||||
WrappedRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: XoRecord['type'] }>,
|
||||
>(
|
||||
idOrUuidOrRef: XoRecord['id'] | WrappedRecord['$ref'] | WrappedRecord['uuid'],
|
||||
defaultValue?: WrappedRecord
|
||||
) => WrappedRecord
|
||||
connectVif(vifId: XoVif['id']): Promise<void>
|
||||
createNetwork(
|
||||
params:
|
||||
| {
|
||||
@@ -130,7 +121,6 @@ export interface Xapi {
|
||||
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
|
||||
pool_emergencyShutdown(): Promise<void>
|
||||
resumeVm(id: XoVm['id']): Promise<void>
|
||||
revertVm(snapshotId: XoVmSnapshot['id']): Promise<void>
|
||||
unpauseVm(id: XoVm['id']): Promise<void>
|
||||
cloneVm(
|
||||
vmId: XoVm['id'],
|
||||
@@ -348,13 +338,4 @@ export interface Xapi {
|
||||
): Promise<XenApiRecord>
|
||||
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
|
||||
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
|
||||
destroySr(id: XoSr['id']): Promise<void>
|
||||
xostor_delete(ref: XenApiSr['$ref']): Promise<void>
|
||||
objects: {
|
||||
indexes: {
|
||||
type: {
|
||||
[XenApiRecord in WrappedXenApiRecord as XenApiRecord['$type']]: Record<XenApiRecord['uuid'], XenApiRecord>
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -491,9 +491,7 @@ export interface XenApiDrTask {
|
||||
uuid: string
|
||||
}
|
||||
|
||||
type XenApiHostCallMethods = TagCallMethods & {
|
||||
<T>(method: 'call_plugin',plugin:string, fn: string, args: Record<string, string>): Promise<T>
|
||||
}
|
||||
type XenApiHostCallMethods = TagCallMethods & {}
|
||||
export interface XenApiHost {
|
||||
$ref: Branded<'host'>
|
||||
address: string
|
||||
|
||||
@@ -44,10 +44,6 @@ import {
|
||||
XenApiVifWrapped,
|
||||
XenApiVmWrapped,
|
||||
XenApiVtpmWrapped,
|
||||
XoAclBasePrivilege,
|
||||
XoAclRole,
|
||||
XoGroupRole,
|
||||
XoUserRole,
|
||||
} from './index.mjs'
|
||||
|
||||
export type XapiConnection = Xapi & {
|
||||
@@ -56,6 +52,7 @@ export type XapiConnection = Xapi & {
|
||||
sessionId: string
|
||||
_url?: { protocol: string; hostname: string; port?: string }
|
||||
}
|
||||
|
||||
type XapiRecordByXapiXoRecord = {
|
||||
gpuGroup: XenApiGpuGroupWrapped
|
||||
host: XenApiHostWrapped
|
||||
@@ -81,42 +78,7 @@ type XapiRecordByXapiXoRecord = {
|
||||
'VM-template': XenApiVmWrapped
|
||||
VTPM: XenApiVtpmWrapped
|
||||
}
|
||||
|
||||
type LicenseProductId =
|
||||
| 'premium'
|
||||
| 'xcpng-enterprise'
|
||||
| 'xcpng-standard'
|
||||
| 'xo-proxy'
|
||||
| 'xosan.trial'
|
||||
| 'xostor'
|
||||
| 'xostor.trial'
|
||||
type LicenseProductType = 'xo' | 'xoproxy' | 'xcpng' | 'xosan' | 'xostor'
|
||||
|
||||
type License = {
|
||||
id: string
|
||||
type: 'license'
|
||||
created: number
|
||||
licenseType: 'unit'
|
||||
productId: LicenseProductId
|
||||
tags: string[]
|
||||
expires?: number
|
||||
boundObjectId?: string
|
||||
bindDate?: number
|
||||
buyer?: { token: string; email: string }
|
||||
history?: { date: number; boundObjectId: string }[]
|
||||
rebound?: number
|
||||
productTypes?: LicenseProductType[]
|
||||
bundleInfo?: { name: string; id: string }
|
||||
}
|
||||
|
||||
export type XoApp = {
|
||||
hooks: EventEmitter
|
||||
_redis: {
|
||||
get(key: string): Promise<string | null>
|
||||
mSet(args: string[]): Promise<unknown>
|
||||
sMembers(key: string): Promise<string[]>
|
||||
sIsMember(key: string, member: string): Promise<boolean>
|
||||
}
|
||||
config: {
|
||||
get<T = string>(path: string): T
|
||||
getOptional(path: string): Record<string, string> | undefined
|
||||
@@ -166,18 +128,7 @@ export type XoApp = {
|
||||
}
|
||||
|
||||
// methods ------------
|
||||
addAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<XoGroupRole>
|
||||
addAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<XoUserRole>
|
||||
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
|
||||
addApiMethod: <A extends unknown[], R>(
|
||||
name: string,
|
||||
method: (...args: A) => Promise<R>,
|
||||
//we use any since it is a legacy call, might be better typed in another pr
|
||||
info: {
|
||||
resolve?: any
|
||||
params?: any
|
||||
}
|
||||
) => () => void // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
authenticateUser: (
|
||||
credentials: { token?: string; username?: string; password?: string },
|
||||
userData?: { ip?: string },
|
||||
@@ -187,18 +138,6 @@ export type XoApp = {
|
||||
checkFeatureAuthorization(featureCode: string): Promise<void>
|
||||
/* connect a server (XCP-ng/XenServer) */
|
||||
connectXenServer(id: XoServer['id']): Promise<void>
|
||||
// TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)
|
||||
createAclV2Privilege(
|
||||
privilege: Omit<XoAclBasePrivilege, 'id'>,
|
||||
options?: {
|
||||
force?: boolean
|
||||
}
|
||||
): Promise<XoAclBasePrivilege>
|
||||
copyAclV2Role(
|
||||
id: XoAclRole['id'],
|
||||
params?: { name?: XoAclRole['name']; description?: XoAclRole['description'] }
|
||||
): Promise<XoAclRole['id']>
|
||||
createAclV2Role(role: { name: XoAclRole['name']; description?: XoAclRole['description'] }): Promise<XoAclRole>
|
||||
createAuthenticationToken(opts: {
|
||||
client?: {
|
||||
id?: string
|
||||
@@ -209,21 +148,11 @@ export type XoApp = {
|
||||
userId: XoUser['id']
|
||||
}): Promise<XoAuthenticationToken>
|
||||
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
|
||||
deleteAclV2GroupRole(groupId: XoGroup['id'], roleId: XoAclRole['id']): Promise<boolean>
|
||||
deleteAclV2Privilege(privilegeId: XoAclBasePrivilege['id'], options?: { force?: boolean }): Promise<boolean>
|
||||
deleteAclV2Role(roleId: XoAclRole['id'], options?: { force?: boolean }): Promise<boolean>
|
||||
deleteGroup(id: XoGroup['id']): Promise<void>
|
||||
deleteUser(id: XoUser['id']): Promise<void>
|
||||
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
|
||||
/* disconnect a server (XCP-ng/XenServer) */
|
||||
disconnectXenServer(id: XoServer['id']): Promise<void>
|
||||
getAclV2Privilege(id: XoAclBasePrivilege['id']): Promise<XoAclBasePrivilege>
|
||||
getAclV2Privileges(): Promise<XoAclBasePrivilege[]>
|
||||
getAclV2RolePrivileges(roleId: XoAclRole['id']): Promise<XoAclBasePrivilege[]>
|
||||
getAclV2Role(id: XoAclRole['id']): Promise<XoAclRole>
|
||||
deleteAclV2UserRole(userId: XoUser['id'], roleId: XoAclRole['id']): Promise<boolean>
|
||||
getAclV2Roles(): Promise<XoAclRole[]>
|
||||
getAclV2UserPrivileges(userId: XoUser['id']): Promise<XoAclBasePrivilege[]>
|
||||
getAllGroups(): Promise<XoGroup[]>
|
||||
getAllProxies(): Promise<XoProxy[]>
|
||||
getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
|
||||
@@ -317,20 +246,6 @@ export type XoApp = {
|
||||
preferences?: Record<string, string>
|
||||
}
|
||||
): Promise<void>
|
||||
updateAclV2Privilege(
|
||||
privilegeId: XoAclBasePrivilege['id'],
|
||||
privilege: XoAclBasePrivilege
|
||||
): Promise<XoAclBasePrivilege>
|
||||
updateAclV2Role(
|
||||
roleId: XoAclRole['id'],
|
||||
role: {
|
||||
name?: XoAclRole['name']
|
||||
description?: XoAclRole['description'] | null
|
||||
},
|
||||
options?: {
|
||||
force?: boolean
|
||||
}
|
||||
): Promise<XoAclRole>
|
||||
updateGroup(
|
||||
id: XoGroup['id'],
|
||||
updates: {
|
||||
@@ -339,12 +254,4 @@ export type XoApp = {
|
||||
): void
|
||||
getAllXapis(): Record<string, XapiConnection>
|
||||
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
|
||||
getLicenses(params?: { productType?: LicenseProductType }): Promise<License[]>
|
||||
bindLicense(params: { licenseId: string; boundObjectId: string }): Promise<License>
|
||||
unbindLicense(params: {
|
||||
productId: LicenseProductId
|
||||
boundObjectId: string
|
||||
licenseId: string
|
||||
data?: Record<string, string>
|
||||
}): Promise<License>
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ import type {
|
||||
VM_POWER_STATE,
|
||||
} from './common.mjs'
|
||||
import type * as CMType from './lib/complex-matcher.mjs'
|
||||
import { XoAclPrivilege, XoAclRole, XoAclSupportedActionsByResource } from './lib/xen-orchestra-acl.mjs'
|
||||
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
|
||||
|
||||
type BaseXapiXo = {
|
||||
@@ -335,7 +334,6 @@ export type XoNetwork = BaseXapiXo & {
|
||||
defaultIsLocked: boolean
|
||||
id: Branded<'network'>
|
||||
insecureNbd?: boolean
|
||||
isBonded: boolean
|
||||
name_description: string
|
||||
name_label: string
|
||||
nbd?: boolean
|
||||
@@ -658,15 +656,13 @@ export type XoTask = {
|
||||
| 'server'
|
||||
| 'task'
|
||||
| 'user'
|
||||
| 'acl-privilege'
|
||||
| 'acl-role'
|
||||
params?: Record<string, unknown>
|
||||
progress?: number
|
||||
type?: string
|
||||
userId?: string
|
||||
[key: string]: unknown | undefined
|
||||
}
|
||||
result?: Record<string, unknown>
|
||||
result: Record<string, unknown>
|
||||
start: number
|
||||
status: 'failure' | 'interrupted' | 'pending' | 'success'
|
||||
tasks?: XoTask[]
|
||||
@@ -680,7 +676,7 @@ export type XoUser = {
|
||||
groups: XoGroup['id'][]
|
||||
id: Branded<'user'>
|
||||
name?: string
|
||||
permission: 'none' | 'admin'
|
||||
permission: string
|
||||
pw_hash?: string
|
||||
preferences: Record<string, string>
|
||||
}
|
||||
@@ -854,10 +850,7 @@ export type XapiXoRecord =
|
||||
| XoVtpm
|
||||
| XoSm
|
||||
|
||||
export type NonXapiXoRecord<
|
||||
TActionsByResource extends XoAclSupportedActionsByResource = never,
|
||||
TResource extends string = never,
|
||||
> =
|
||||
export type NonXapiXoRecord =
|
||||
| AnyXoBackupArchive
|
||||
| AnyXoJob
|
||||
| AnyXoLog
|
||||
@@ -871,13 +864,8 @@ export type NonXapiXoRecord<
|
||||
| XoServer
|
||||
| XoTask
|
||||
| XoUser
|
||||
| XoAclRole
|
||||
| XoAclPrivilege<TActionsByResource, TResource>
|
||||
|
||||
export type XoRecord<
|
||||
TActionsByResource extends XoAclSupportedActionsByResource = never,
|
||||
TResource extends string = never,
|
||||
> = XapiXoRecord | NonXapiXoRecord<TActionsByResource, TResource>
|
||||
export type XoRecord = XapiXoRecord | NonXapiXoRecord
|
||||
|
||||
export type AnyXoVm = XoVm | XoVmSnapshot | XoVmTemplate | XoVmController
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../../scripts/npmignore
|
||||
@@ -1,16 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
import eslint from '@eslint/js'
|
||||
import tseslint from 'typescript-eslint'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import path from 'node:path'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
},
|
||||
}
|
||||
})
|
||||
@@ -1,47 +0,0 @@
|
||||
{
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"main": "dist/index.mjs",
|
||||
"name": "@xen-orchestra/acl",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/acl",
|
||||
"version": "1.0.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">= 20.18"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.mts",
|
||||
"default": "./dist/index.mjs"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && node scripts/generate-types.mjs",
|
||||
"clean": "rimraf dist/",
|
||||
"dev": "tsc --watch",
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "npm run build && node --test ./dist/**/*.test.mjs",
|
||||
"prebuild": "npm run clean",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"@vates/types": "^1.24.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.23.0"
|
||||
},
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@xen-orchestra/acl",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"dependencies": {
|
||||
"complex-matcher": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// AUTO-GENERATES dist/generated/privilege-types.d.mts
|
||||
// Run via `npm run build` (after tsc).
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../dist/actions/index.mjs'
|
||||
import { getActionStrings } from '../dist/index.mjs'
|
||||
|
||||
function toPascalCase(str) {
|
||||
return str
|
||||
.split(/[-_]/)
|
||||
.map(s => s.charAt(0).toUpperCase() + s.slice(1))
|
||||
.join('')
|
||||
}
|
||||
|
||||
const resources = Object.keys(SUPPORTED_ACTIONS_BY_RESOURCE)
|
||||
|
||||
const typeBlocks = resources.map(resource => {
|
||||
const actionUnion = getActionStrings(resource)
|
||||
.map(action => `'${action}'`)
|
||||
.join(' | ')
|
||||
return `export type ${toPascalCase(resource)}Privilege = {
|
||||
id: XoAclBasePrivilege['id']
|
||||
resource: '${resource}'
|
||||
action: ${actionUnion}
|
||||
selector?: XoAclBasePrivilege['selector']
|
||||
effect: XoAclBasePrivilege['effect']
|
||||
roleId: XoAclBasePrivilege['roleId']
|
||||
}`
|
||||
})
|
||||
|
||||
const anyUnion = resources.map(resource => ` | ${toPascalCase(resource)}Privilege`).join('\n')
|
||||
|
||||
const types = `\
|
||||
// -----------------------------------------------------------------------
|
||||
// AUTO-GENERATED — do not edit manually.
|
||||
// Regenerated on every \`npm run build\`.
|
||||
// -----------------------------------------------------------------------
|
||||
import type { XoAclBasePrivilege } from '@vates/types'
|
||||
|
||||
|
||||
${typeBlocks.join('\n\n')}
|
||||
|
||||
export type AnyPrivilege =
|
||||
${anyUnion}
|
||||
`
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const outDir = join(root, 'dist/generated')
|
||||
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
writeFileSync(join(outDir, 'privilege-types.d.mts'), types, 'utf-8')
|
||||
@@ -1,11 +0,0 @@
|
||||
export default {
|
||||
create: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
update: {
|
||||
action: true,
|
||||
effect: true,
|
||||
resource: true,
|
||||
selector: true,
|
||||
},
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
export default {
|
||||
create: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
update: {
|
||||
description: true,
|
||||
groups: true,
|
||||
name: true,
|
||||
users: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
export default {
|
||||
create: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
update: {
|
||||
name: true,
|
||||
users: true,
|
||||
},
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
export default {
|
||||
'allow-vm': true,
|
||||
disable: true,
|
||||
enable: true,
|
||||
evacuate: true,
|
||||
export: {
|
||||
logs: true,
|
||||
},
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
import { default as aclPrivilege } from './acl-privilege.mjs'
|
||||
import { default as aclRole } from './acl-role.mjs'
|
||||
import { default as alarm } from './alarm.mjs'
|
||||
import { default as backupArchive } from './backup-archive.mjs'
|
||||
import { default as backupJob } from './backup-job.mjs'
|
||||
import { default as backupLog } from './backup-log.mjs'
|
||||
import { default as backupRepository } from './backup-repository.mjs'
|
||||
import { default as gpuGroup } from './gpuGroup.mjs'
|
||||
import { default as group } from './group.mjs'
|
||||
import { default as host } from './host.mjs'
|
||||
import { default as message } from './message.mjs'
|
||||
import { default as network } from './network.mjs'
|
||||
import { default as pbd } from './pbd.mjs'
|
||||
import { default as pci } from './pci.mjs'
|
||||
import { default as pgpu } from './pgpu.mjs'
|
||||
import { default as pif } from './pif.mjs'
|
||||
import { default as pool } from './pool.mjs'
|
||||
import { default as proxy } from './proxy.mjs'
|
||||
import { default as restoreLog } from './restore-log.mjs'
|
||||
import { default as schedule } from './schedule.mjs'
|
||||
import { default as server } from './server.mjs'
|
||||
import { default as sm } from './sm.mjs'
|
||||
import { default as sr } from './sr.mjs'
|
||||
import { default as task } from './task.mjs'
|
||||
import { default as user } from './user.mjs'
|
||||
import { default as vbd } from './vbd.mjs'
|
||||
import { default as vdiSnapshot } from './vdi-snapshot.mjs'
|
||||
import { default as vdiUnmanaged } from './vdi-unmanaged.mjs'
|
||||
import { default as vdi } from './vdi.mjs'
|
||||
import { default as vgpu } from './vgpu.mjs'
|
||||
import { default as vgpuType } from './vgpuType.mjs'
|
||||
import { default as vif } from './vif.mjs'
|
||||
import { default as vmController } from './vm-controller.mjs'
|
||||
import { default as vmSnapshot } from './vm-snapshot.mjs'
|
||||
import { default as vmTemplate } from './vm-template.mjs'
|
||||
import { default as vm } from './vm.mjs'
|
||||
import { default as vtpm } from './vtpm.mjs'
|
||||
|
||||
/**
|
||||
* value without sub action like 'shutdown' mean: 'shutdown:*' (so shutdown mean you can shutdown:clean and shutdown:hard)
|
||||
*/
|
||||
export const SUPPORTED_ACTIONS_BY_RESOURCE = {
|
||||
'acl-privilege': aclPrivilege,
|
||||
'acl-role': aclRole,
|
||||
alarm,
|
||||
'backup-archive': backupArchive,
|
||||
'backup-job': backupJob,
|
||||
'backup-log': backupLog,
|
||||
'backup-repository': backupRepository,
|
||||
group,
|
||||
gpuGroup,
|
||||
host,
|
||||
message,
|
||||
network,
|
||||
pbd,
|
||||
pci,
|
||||
pgpu,
|
||||
pif,
|
||||
pool,
|
||||
proxy,
|
||||
'restore-log': restoreLog,
|
||||
schedule,
|
||||
server,
|
||||
sm,
|
||||
sr,
|
||||
task,
|
||||
user,
|
||||
vbd,
|
||||
'vdi-snapshot': vdiSnapshot,
|
||||
'vdi-unmanaged': vdiUnmanaged,
|
||||
vdi,
|
||||
vgpu,
|
||||
vgpuType,
|
||||
vif,
|
||||
'vm-controller': vmController,
|
||||
'vm-snapshot': vmSnapshot,
|
||||
'vm-template': vmTemplate,
|
||||
vm,
|
||||
vtpm,
|
||||
} as const
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
export default {
|
||||
create: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
update: {
|
||||
management: true,
|
||||
},
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
export default {
|
||||
create: {
|
||||
network: true,
|
||||
vm: true,
|
||||
},
|
||||
'emergency-shutdown': true,
|
||||
read: true,
|
||||
'rolling-reboot': true,
|
||||
'rolling-update': true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
run: true,
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
export default {
|
||||
connect: true,
|
||||
create: true,
|
||||
delete: true,
|
||||
disconnect: true,
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
export default {
|
||||
delete: true,
|
||||
import: {
|
||||
vdi: true,
|
||||
vm: true,
|
||||
},
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
export default {
|
||||
abort: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
export default {
|
||||
create: true,
|
||||
delete: true,
|
||||
read: true,
|
||||
update: {
|
||||
name: true,
|
||||
password: true,
|
||||
permission: true,
|
||||
preferences: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
export default {
|
||||
boot: true,
|
||||
create: true,
|
||||
delete: true,
|
||||
'export-content': true,
|
||||
'import-content': true,
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
create: true,
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
export default {
|
||||
delete: true,
|
||||
export: true,
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
export default {
|
||||
delete: true,
|
||||
export: true,
|
||||
instantiate: true,
|
||||
read: true,
|
||||
update: {
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
export default {
|
||||
delete: true,
|
||||
export: true,
|
||||
pause: true,
|
||||
read: true,
|
||||
reboot: {
|
||||
clean: true,
|
||||
hard: true,
|
||||
},
|
||||
resume: true,
|
||||
'revert-snapshot': true,
|
||||
shutdown: {
|
||||
clean: true,
|
||||
hard: true,
|
||||
},
|
||||
snapshot: true,
|
||||
start: true,
|
||||
suspend: true,
|
||||
unpause: true,
|
||||
update: {
|
||||
datasources: true,
|
||||
tags: true,
|
||||
},
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export default {
|
||||
read: true,
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
import * as CM from 'complex-matcher'
|
||||
import type { XoAclPrivilege, XoAclSupportedActions, XoAclSupportedResource } from '@vates/types/lib/xen-orchestra/acl'
|
||||
|
||||
import { SUPPORTED_ACTIONS_BY_RESOURCE } from '../actions/index.mjs'
|
||||
|
||||
export type SupportedActionsByResource = typeof SUPPORTED_ACTIONS_BY_RESOURCE
|
||||
export type SupportedResource = XoAclSupportedResource<SupportedActionsByResource>
|
||||
export type SupportedActions<T extends SupportedResource> = XoAclSupportedActions<SupportedActionsByResource, T>
|
||||
export type TPrivilege<T extends SupportedResource> = XoAclPrivilege<SupportedActionsByResource, T>
|
||||
|
||||
export class Privilege<T extends SupportedResource> {
|
||||
#action: TPrivilege<T>['action']
|
||||
#selector?: (object: object) => boolean
|
||||
#resource: TPrivilege<T>['resource']
|
||||
#effect: TPrivilege<T>['effect']
|
||||
|
||||
constructor({
|
||||
action,
|
||||
selector,
|
||||
resource,
|
||||
effect,
|
||||
}: {
|
||||
action: TPrivilege<T>['action']
|
||||
selector?: TPrivilege<T>['selector']
|
||||
resource: TPrivilege<T>['resource']
|
||||
effect: TPrivilege<T>['effect']
|
||||
}) {
|
||||
Privilege.checkActionIsValid(resource, action)
|
||||
|
||||
this.#action = action
|
||||
this.#selector = selector !== undefined ? CM.parse(selector).createPredicate() : undefined
|
||||
this.#resource = resource
|
||||
this.#effect = effect
|
||||
}
|
||||
|
||||
get effect() {
|
||||
return this.#effect
|
||||
}
|
||||
|
||||
#matchAction<Resource extends SupportedResource = T>(action: SupportedActions<Resource>) {
|
||||
// read:name_label - read:name_label
|
||||
if (this.#action === action) {
|
||||
return true
|
||||
}
|
||||
|
||||
const [namespaceToMatch, actionToMatch] = action.split(':')
|
||||
const [thisNamespace, thisAction] = this.#action.split(':')
|
||||
|
||||
if (thisNamespace === '*') {
|
||||
return true
|
||||
}
|
||||
|
||||
// update:vm - read:name_label
|
||||
if (thisNamespace !== namespaceToMatch) {
|
||||
return false
|
||||
}
|
||||
|
||||
// read - read:name_label
|
||||
if (thisAction === undefined) {
|
||||
return true
|
||||
}
|
||||
// read:name_decription - read:name_label
|
||||
if (thisAction !== actionToMatch) {
|
||||
return false
|
||||
}
|
||||
|
||||
throw new Error(`Unable to verify if ${this.#action} match ${action} `)
|
||||
}
|
||||
|
||||
#matchSelector(object: object) {
|
||||
if (this.#selector === undefined) {
|
||||
return true
|
||||
}
|
||||
|
||||
return this.#selector(object)
|
||||
}
|
||||
|
||||
#matchResource(resource: string): resource is SupportedResource {
|
||||
return resource === this.#resource
|
||||
}
|
||||
|
||||
match<Resource extends SupportedResource = T>(constraint: {
|
||||
action: SupportedActions<Resource>
|
||||
resource: Resource
|
||||
object: object
|
||||
}): boolean {
|
||||
return (
|
||||
this.#matchResource(constraint.resource) &&
|
||||
this.#matchAction(constraint.action) &&
|
||||
this.#matchSelector(constraint.object)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if action not supported
|
||||
*/
|
||||
static checkActionIsValid<T extends SupportedResource>(resource: T, action: SupportedActions<T>) {
|
||||
const supportedActions = SUPPORTED_ACTIONS_BY_RESOURCE[resource]
|
||||
if (supportedActions === undefined) {
|
||||
throw new Error(`${resource} resource not supported`)
|
||||
}
|
||||
|
||||
if (action === '*') {
|
||||
return
|
||||
}
|
||||
|
||||
const segments = action.split(':')
|
||||
let _action: object | undefined = undefined
|
||||
segments.forEach(segment => {
|
||||
_action = (_action ?? supportedActions)[segment]
|
||||
|
||||
if (_action === undefined) {
|
||||
throw new Error(
|
||||
`${action} action not supported for the resource: ${resource}. See ${JSON.stringify(supportedActions)}`
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
// DO NOT EDIT
|
||||
// Overwritten by scripts/generate-types.mjs after running npm run build
|
||||
// scripts/generate-types.mjs will replace this file to automatically generate and export all privileged types.
|
||||
// This makes external use easier because it's not necessary to resolve the type every time.
|
||||
|
||||
import type { Privilege, SupportedResource } from '../index.mjs'
|
||||
|
||||
export type AnyPrivilege = {
|
||||
[Resource in SupportedResource]: Privilege<Resource>
|
||||
}[SupportedResource]
|
||||
@@ -1,167 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import type { XoUser } from '@vates/types/xo'
|
||||
|
||||
import { SUPPORTED_ACTIONS_BY_RESOURCE } from './actions/index.mjs'
|
||||
import {
|
||||
Privilege as CPrivilege,
|
||||
TPrivilege as Privilege,
|
||||
SupportedActions,
|
||||
SupportedActionsByResource,
|
||||
SupportedResource,
|
||||
} from './class/privilege.mjs'
|
||||
|
||||
// Export all types constructed using SUPPORTED_ACTIONS_BY_RESOURCE
|
||||
export type { Privilege, SupportedActions, SupportedActionsByResource, SupportedResource }
|
||||
export { SUPPORTED_ACTIONS_BY_RESOURCE }
|
||||
|
||||
export type * from './generated/privilege-types.mjs'
|
||||
|
||||
import { AnyPrivilege } from './generated/privilege-types.mjs'
|
||||
|
||||
export type AnyPrivilegeOnParam = {
|
||||
[Resource in SupportedResource]: {
|
||||
user: XoUser
|
||||
resource: Resource
|
||||
action: SupportedActions<Resource>
|
||||
objects: object | object[]
|
||||
}
|
||||
}[SupportedResource]
|
||||
|
||||
export function hasPrivilegeOn<T extends SupportedResource>({
|
||||
user,
|
||||
action,
|
||||
resource,
|
||||
objects,
|
||||
userPrivileges,
|
||||
}: {
|
||||
user: XoUser
|
||||
resource: T
|
||||
action: SupportedActions<T>
|
||||
objects: object | object[]
|
||||
userPrivileges: AnyPrivilege[]
|
||||
}) {
|
||||
// Function that will be called outside of the module
|
||||
// We cannot be sure types are respected
|
||||
assert.strictEqual(typeof user?.permission, 'string')
|
||||
assert.strictEqual(typeof action, 'string')
|
||||
assert.strictEqual(typeof resource, 'string')
|
||||
assert.strictEqual(objects === undefined, false)
|
||||
|
||||
CPrivilege.checkActionIsValid(resource, action as SupportedActions<typeof resource>)
|
||||
|
||||
if (user.permission === 'admin') {
|
||||
return true
|
||||
}
|
||||
|
||||
const effectsByObject = new Map<object, Privilege<T>['effect'][]>()
|
||||
if (Array.isArray(objects)) {
|
||||
objects.forEach(obj => effectsByObject.set(obj, []))
|
||||
} else {
|
||||
effectsByObject.set(objects, [])
|
||||
}
|
||||
|
||||
for (const userPrivilege of userPrivileges) {
|
||||
const privilege = new CPrivilege(userPrivilege as Privilege<typeof userPrivilege.resource>)
|
||||
|
||||
for (const [object, effects] of effectsByObject.entries()) {
|
||||
if (!privilege.match({ action, resource, object })) {
|
||||
continue
|
||||
}
|
||||
effects.push(privilege.effect)
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
Array.from(effectsByObject.values()).some(
|
||||
effects => effects.length === 0 || effects.some(effect => effect === 'deny')
|
||||
)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
export function getMissingPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
|
||||
return params
|
||||
.filter(
|
||||
param =>
|
||||
!hasPrivilegeOn({
|
||||
user: param.user,
|
||||
resource: param.resource,
|
||||
action: param.action as SupportedActions<typeof param.resource>,
|
||||
objects: param.objects,
|
||||
userPrivileges,
|
||||
})
|
||||
)
|
||||
.map(({ action, objects, resource }) => {
|
||||
let objectIds: unknown[] | undefined
|
||||
let objectId: unknown | undefined
|
||||
|
||||
if (Array.isArray(objects)) {
|
||||
if (objects.length > 1) {
|
||||
objectIds = []
|
||||
objects.forEach(obj => {
|
||||
if ('id' in obj) {
|
||||
objectIds!.push(obj.id)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
objects = objects?.[0] ?? {}
|
||||
}
|
||||
}
|
||||
if (!Array.isArray(objects)) {
|
||||
objectId = 'id' in objects ? objects.id : undefined
|
||||
}
|
||||
|
||||
return {
|
||||
objectId,
|
||||
objectIds,
|
||||
action,
|
||||
resource,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function hasPrivileges(params: AnyPrivilegeOnParam[], userPrivileges: AnyPrivilege[]) {
|
||||
return getMissingPrivileges(params, userPrivileges).length === 0
|
||||
}
|
||||
|
||||
export function filterObjectsWithPrivilege<Resource extends SupportedResource, Object extends object>(param: {
|
||||
user: XoUser
|
||||
resource: Resource
|
||||
action: SupportedActions<Resource>
|
||||
objects: Object[]
|
||||
userPrivileges: AnyPrivilege[]
|
||||
}) {
|
||||
return param.objects.filter(obj => hasPrivilegeOn({ ...param, objects: obj }))
|
||||
}
|
||||
|
||||
// Utils
|
||||
|
||||
/**
|
||||
* Mirror GetKeysRecursively from @vates/types
|
||||
*/
|
||||
export function getActionStrings<Resource extends SupportedResource, Actions = SupportedActions<Resource>>(
|
||||
resource: Resource
|
||||
): Actions[] {
|
||||
type NestedActions = { [key: string]: boolean | NestedActions }
|
||||
|
||||
function collectActions(obj: NestedActions, prefix?: Actions): Actions[] {
|
||||
const actions: Actions[] = ['*' as Actions]
|
||||
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const action = (prefix !== undefined ? `${prefix}:${key}` : key) as Actions
|
||||
actions.push(action)
|
||||
|
||||
if (typeof value === 'boolean') {
|
||||
continue
|
||||
}
|
||||
|
||||
actions.push(...collectActions(value, action))
|
||||
}
|
||||
|
||||
return actions
|
||||
}
|
||||
|
||||
return collectActions(SUPPORTED_ACTIONS_BY_RESOURCE[resource])
|
||||
}
|
||||
@@ -1,378 +0,0 @@
|
||||
import { test, suite } from 'node:test'
|
||||
import assert from 'node:assert'
|
||||
import type { Branded } from '@vates/types/common'
|
||||
import type { XoUser } from '@vates/types/xo'
|
||||
|
||||
import { Privilege } from '../class/privilege.mjs'
|
||||
import type { Privilege as TPrivilege } from '../index.mjs'
|
||||
import {
|
||||
filterObjectsWithPrivilege,
|
||||
getActionStrings,
|
||||
getMissingPrivileges,
|
||||
hasPrivilegeOn,
|
||||
hasPrivileges,
|
||||
} from '../index.mjs'
|
||||
|
||||
suite('Privilege.checkActionIsValid behavior', () => {
|
||||
test('throw on invalid resource', () => {
|
||||
// @ts-expect-error test invalid action
|
||||
assert.throws(() => Privilege.checkActionIsValid('foo', 'read'))
|
||||
})
|
||||
|
||||
test('throw on invalid action', () => {
|
||||
// @ts-expect-error test invalid action
|
||||
assert.throws(() => Privilege.checkActionIsValid('vm', 'foo:vm'))
|
||||
})
|
||||
|
||||
test('should work', () => {
|
||||
assert.doesNotThrow(() => Privilege.checkActionIsValid('vm', 'read'))
|
||||
})
|
||||
})
|
||||
|
||||
suite('Privilege.match behavior', () => {
|
||||
const privilege = new Privilege({ action: 'read', resource: 'vm', effect: 'allow' })
|
||||
const object = { id: 1, name_label: 'foo' }
|
||||
|
||||
test('Should not match because of the action', () => {
|
||||
assert.strictEqual(privilege.match({ resource: 'vm', action: 'start', object }), false)
|
||||
})
|
||||
|
||||
test('Should not match because of the resource', () => {
|
||||
assert.strictEqual(privilege.match({ resource: 'host', action: 'read', object }), false)
|
||||
})
|
||||
|
||||
test('Should match', () => {
|
||||
assert.strictEqual(privilege.match({ resource: 'vm', action: 'read', object }), true)
|
||||
})
|
||||
|
||||
test('Should match even if the action is not equal to the action privilege', () => {
|
||||
const shutdownPrivilege = new Privilege({ action: 'shutdown', resource: 'vm', effect: 'allow' })
|
||||
assert.strictEqual(shutdownPrivilege.match({ resource: 'vm', action: 'shutdown:clean', object }), true)
|
||||
})
|
||||
|
||||
test('Should match because the action is *', () => {
|
||||
const vmPrivilege = new Privilege({ action: '*', resource: 'vm', effect: 'allow' })
|
||||
assert.strictEqual(vmPrivilege.match({ action: 'reboot:hard', resource: 'vm', object: object }), true)
|
||||
})
|
||||
|
||||
test('Should not match due to insufficient privilege', () => {
|
||||
const cleanShutdownPrivilege = new Privilege({ action: 'shutdown:clean', resource: 'vm', effect: 'allow' })
|
||||
assert.strictEqual(cleanShutdownPrivilege.match({ resource: 'vm', action: 'shutdown', object }), false)
|
||||
})
|
||||
})
|
||||
|
||||
suite('ACL V2 behavior', async () => {
|
||||
// === Setup ===
|
||||
const admin = {
|
||||
id: '1' as XoUser['id'],
|
||||
groups: [],
|
||||
permission: 'admin',
|
||||
email: 'admin@admin.net',
|
||||
preferences: {},
|
||||
} satisfies XoUser
|
||||
|
||||
const user = {
|
||||
id: '2' as XoUser['id'],
|
||||
groups: [],
|
||||
permission: 'none',
|
||||
email: 'user@user.net',
|
||||
preferences: {},
|
||||
} satisfies XoUser
|
||||
|
||||
const privilegeId = 'fake-priv-test-id' as Branded<'acl-v2-privilege'>
|
||||
const roleId = 'fake-role-test-id' as Branded<'acl-v2-role'>
|
||||
|
||||
// === Objects
|
||||
const poolWithoutPrivilege = { id: 'pool-id' }
|
||||
const alpineVm = { id: 'alpine', creation: { creator: admin.id } }
|
||||
const almaVm = { id: 'alma', creation: { creator: user.id } }
|
||||
const xoaVm = { id: 'xoa' }
|
||||
const template = { id: 'template-id', $pool: '1' }
|
||||
const templateDeny = { id: 'template-deny-id', $pool: '2' }
|
||||
const vdis = [{ $SR: '2' }, { $SR: '2' }]
|
||||
const vdisMultipleSrs = [{ $SR: '1' }, { $SR: '2' }]
|
||||
const vifs = [{ $network: '1' }]
|
||||
const iso = { id: 'iso-vdi-id', $SR: '3' }
|
||||
const hostAffinity = { id: 'affinity-host-id', $pool: '1' }
|
||||
|
||||
// === DB Privileges
|
||||
const vmRead = { id: privilegeId, action: 'read', resource: 'vm', effect: 'allow', roleId } satisfies TPrivilege<'vm'>
|
||||
const vmCleanShutdown = {
|
||||
id: privilegeId,
|
||||
action: 'shutdown:clean',
|
||||
resource: 'vm',
|
||||
selector: `creation:creator:${user.id}`,
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vm'>
|
||||
const xoaVmDenyRead = {
|
||||
id: privilegeId,
|
||||
action: 'read',
|
||||
resource: 'vm',
|
||||
effect: 'deny',
|
||||
selector: `id: ${xoaVm.id}`,
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vm'>
|
||||
const vdiCreate = {
|
||||
id: privilegeId,
|
||||
action: 'create',
|
||||
resource: 'vdi',
|
||||
selector: '$SR:2',
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vdi'>
|
||||
const vifCreate = {
|
||||
id: privilegeId,
|
||||
action: 'create',
|
||||
resource: 'vif',
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vif'>
|
||||
const templateInstantiate = {
|
||||
id: privilegeId,
|
||||
action: 'instantiate',
|
||||
resource: 'vm-template',
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vm-template'>
|
||||
const templateInstantiateDeny = {
|
||||
id: privilegeId,
|
||||
action: 'instantiate',
|
||||
resource: 'vm-template',
|
||||
selector: '$pool:2',
|
||||
effect: 'deny',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vm-template'>
|
||||
const isoVdiUse = {
|
||||
id: privilegeId,
|
||||
action: 'boot',
|
||||
resource: 'vdi',
|
||||
selector: '$SR:3',
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'vdi'>
|
||||
const hostVm = {
|
||||
id: privilegeId,
|
||||
action: 'allow-vm',
|
||||
resource: 'host',
|
||||
effect: 'allow',
|
||||
roleId,
|
||||
} satisfies TPrivilege<'host'>
|
||||
|
||||
const allPrivileges = [
|
||||
vmRead,
|
||||
vmCleanShutdown,
|
||||
xoaVmDenyRead,
|
||||
vdiCreate,
|
||||
vifCreate,
|
||||
templateInstantiate,
|
||||
templateInstantiateDeny,
|
||||
isoVdiUse,
|
||||
hostVm,
|
||||
]
|
||||
|
||||
// === Test ===
|
||||
suite('hasPrivilegeOn behavior', () => {
|
||||
test('Should always return true for admin', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn<'pool'>({
|
||||
user: admin,
|
||||
action: 'read',
|
||||
resource: 'pool',
|
||||
objects: poolWithoutPrivilege,
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('Should return false by default for non admin', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({
|
||||
user,
|
||||
action: 'read',
|
||||
resource: 'pool',
|
||||
objects: poolWithoutPrivilege,
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('Should return true if user has privilege', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: alpineVm, userPrivileges: allPrivileges }),
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
test('Should return false if user have a both deny and allow privileges', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({ user, action: 'read', resource: 'vm', objects: xoaVm, userPrivileges: allPrivileges }),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('Should handle multiple and single object', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({
|
||||
user,
|
||||
action: 'read',
|
||||
resource: 'vm',
|
||||
objects: [alpineVm, almaVm],
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
true
|
||||
)
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({
|
||||
user,
|
||||
action: 'read',
|
||||
resource: 'vm',
|
||||
objects: [alpineVm, almaVm, xoaVm],
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
test('Should allow clean shutdown only own VMs', () => {
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({
|
||||
user,
|
||||
action: 'shutdown:clean',
|
||||
resource: 'vm',
|
||||
objects: almaVm,
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
true
|
||||
)
|
||||
assert.strictEqual(
|
||||
hasPrivilegeOn({
|
||||
user,
|
||||
action: 'shutdown:clean',
|
||||
resource: 'vm',
|
||||
objects: [almaVm, alpineVm],
|
||||
userPrivileges: allPrivileges,
|
||||
}),
|
||||
false
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
suite('getMissingPrivileges behavior', () => {
|
||||
test('Should return empty array if no missing privileges', () => {
|
||||
const missingPrivileges = getMissingPrivileges(
|
||||
[
|
||||
{ user, action: 'shutdown:clean', resource: 'vm', objects: almaVm },
|
||||
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
|
||||
{ user, action: 'create', resource: 'vdi', objects: vdis },
|
||||
{ user, action: 'create', resource: 'vif', objects: vifs },
|
||||
{ user, action: 'boot', resource: 'vdi', objects: iso },
|
||||
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
|
||||
],
|
||||
allPrivileges
|
||||
)
|
||||
|
||||
assert.strictEqual(missingPrivileges.length, 0)
|
||||
})
|
||||
|
||||
test('Should return only missing privileges', () => {
|
||||
const missingPrivileges = getMissingPrivileges(
|
||||
[
|
||||
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
|
||||
{ user, action: 'create', resource: 'vdi', objects: vdis },
|
||||
{ user, action: 'create', resource: 'vif', objects: vifs },
|
||||
{ user, action: 'boot', resource: 'vdi', objects: iso },
|
||||
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
|
||||
{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege },
|
||||
{ user, action: '*', resource: 'pool', objects: poolWithoutPrivilege },
|
||||
],
|
||||
allPrivileges
|
||||
)
|
||||
|
||||
assert.strictEqual(missingPrivileges.length, 2)
|
||||
})
|
||||
|
||||
test('Should return objectId if only one object in the missing privilege', () => {
|
||||
const missingPrivileges = getMissingPrivileges(
|
||||
[{ user, action: 'read', resource: 'pool', objects: poolWithoutPrivilege }],
|
||||
allPrivileges
|
||||
)
|
||||
|
||||
assert.strictEqual(missingPrivileges.length, 1)
|
||||
assert.notStrictEqual(missingPrivileges[0]!.objectId, undefined)
|
||||
})
|
||||
|
||||
test('Should return objectIds if multiple objects in the missing privilege', () => {
|
||||
const missingPrivileges = getMissingPrivileges(
|
||||
[{ user, action: 'read', resource: 'pool', objects: [poolWithoutPrivilege, poolWithoutPrivilege] }],
|
||||
allPrivileges
|
||||
)
|
||||
|
||||
assert.strictEqual(missingPrivileges.length, 1)
|
||||
assert.notStrictEqual(missingPrivileges[0]!.objectIds, undefined)
|
||||
})
|
||||
})
|
||||
|
||||
suite('hasPrivileges behavior', () => {
|
||||
test('Should return false if at least one privilege deny', () => {
|
||||
const result = hasPrivileges(
|
||||
[
|
||||
{ user, action: 'instantiate', resource: 'vm-template', objects: templateDeny },
|
||||
{ user, action: 'create', resource: 'vdi', objects: vdis },
|
||||
{ user, action: 'create', resource: 'vif', objects: vifs },
|
||||
{ user, action: 'boot', resource: 'vdi', objects: iso },
|
||||
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
|
||||
],
|
||||
allPrivileges
|
||||
)
|
||||
assert.strictEqual(result, false)
|
||||
})
|
||||
test("Should return false if at least one privilege didn't match", () => {
|
||||
const result = hasPrivileges(
|
||||
[
|
||||
{ user, action: 'instantiate', resource: 'vm-template', objects: template },
|
||||
{ user, action: 'create', resource: 'vdi', objects: vdisMultipleSrs },
|
||||
{ user, action: 'create', resource: 'vif', objects: vifs },
|
||||
{ user, action: 'boot', resource: 'vdi', objects: iso },
|
||||
{ user, action: 'allow-vm', resource: 'host', objects: hostAffinity },
|
||||
],
|
||||
allPrivileges
|
||||
)
|
||||
assert.strictEqual(result, false)
|
||||
})
|
||||
})
|
||||
|
||||
suite('filterObjectsWithPrivilege behavior', () => {
|
||||
test('Should return only objects that match privileges', () => {
|
||||
let vms = [almaVm, alpineVm, xoaVm, xoaVm]
|
||||
assert.strictEqual(vms.length, 4)
|
||||
vms = filterObjectsWithPrivilege({
|
||||
user,
|
||||
action: 'read',
|
||||
resource: 'vm',
|
||||
objects: vms,
|
||||
userPrivileges: allPrivileges,
|
||||
})
|
||||
assert.strictEqual(vms.length, 2)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
suite('getActionStrings', () => {
|
||||
test('returns flat actions for a resource without nesting', () => {
|
||||
const actions = getActionStrings('host')
|
||||
assert.ok(actions.includes('read'))
|
||||
// @ts-ignore A TypeScript error is thrown because `foo` is not of type `SupportedActions<'vm'>`.
|
||||
// To test for a negative value, the error is ignored.
|
||||
assert.strictEqual(actions.includes('foo'), false)
|
||||
})
|
||||
|
||||
test('parent action is included alongside its children', () => {
|
||||
const actions = getActionStrings('vm')
|
||||
assert.ok(actions.includes('shutdown'))
|
||||
assert.ok(actions.includes('shutdown:clean'))
|
||||
assert.ok(actions.includes('shutdown:hard'))
|
||||
})
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
|
||||
"strict": true,
|
||||
"module": "nodenext",
|
||||
"lib": ["ESNext"],
|
||||
"moduleResolution": "nodenext",
|
||||
"target": "ES2022",
|
||||
"skipLibCheck": true,
|
||||
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
|
||||
"noImplicitAny": false, // otherwise it will throw an error when importing packages without type definition
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"declaration": true
|
||||
},
|
||||
"include": ["./src/**/*"],
|
||||
"exclude": ["./node_modules", "./dist"]
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
../../scripts/npmignore
|
||||
@@ -1,21 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
import eslint from '@eslint/js'
|
||||
import tseslint from 'typescript-eslint'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import path from 'node:path'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
export default tseslint.config(eslint.configs.recommended, tseslint.configs.recommended, {
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
},
|
||||
},
|
||||
ignores: ['dist'],
|
||||
rules: {
|
||||
// Allow empty interface for recursive generic typing (E.g. interface XenApiNetworkWrapped)
|
||||
'@typescript-eslint/no-empty-object-type': 'off',
|
||||
},
|
||||
})
|
||||
@@ -1,55 +0,0 @@
|
||||
{
|
||||
"name": "@xen-orchestra/backup-archive",
|
||||
"version": "1.0.0",
|
||||
"description": "Helper to handle generic backup archives",
|
||||
"license": "ISC",
|
||||
"author": "",
|
||||
"type": "module",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": "./dist/index.mjs",
|
||||
"./disks": "./dist/disks/index.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"@types/node": "^22.3.0",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.23.0",
|
||||
"fs-extra": "^11.1.0",
|
||||
"sinon": "^18.0.0",
|
||||
"tar-stream": "^3.1.8",
|
||||
"@types/tar-stream": "^3.1.4",
|
||||
"tmp": "^0.2.1",
|
||||
"@vates/types": "^1.21.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/fs": "^4.6.7",
|
||||
"@xen-orchestra/disk-transform": "^1.2.3",
|
||||
"@vates/task": "0.7.0",
|
||||
"golike-defer": "0.5.1",
|
||||
"uuid": "^9.0.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"@vates/async-each": "^1.0.1",
|
||||
"vhd-lib": "^4.14.7"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "yarn run clean && tsc",
|
||||
"dev": "tsc --watch",
|
||||
"postversion": "npm publish --access public",
|
||||
"test-integration": "yarn run clean && tsc && node --test **/*.integ.mjs",
|
||||
"clean": "rimraf dist/"
|
||||
},
|
||||
"private": true,
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backup-archive",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@xen-orchestra/backup-archive",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.17"
|
||||
}
|
||||
}
|
||||
@@ -1,409 +0,0 @@
|
||||
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
|
||||
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
|
||||
import {
|
||||
MergeRemoteDisk,
|
||||
openDisk,
|
||||
openDisposableDisk,
|
||||
RemoteDisk,
|
||||
isDisk,
|
||||
openDiskChain,
|
||||
} from '@xen-orchestra/backup-archive/disks'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import {
|
||||
CleanResult,
|
||||
DEFAULT_MERGE_CONCURRENCY,
|
||||
DEFAULT_REMOVE_CONCURRENCY,
|
||||
ResolvedBackupCleanOptions,
|
||||
} from './VmBackup.types.mjs'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { Task } from '@vates/task'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
const defaultMergeLimiter = limitConcurrency(1)
|
||||
/**
|
||||
* Tracks the disk chain for a single VDI across all backup snapshots.
|
||||
* Owns merge and deletion decisions for its chain given which disks are still
|
||||
* referenced by active backups (accumulated via addActiveDiskPath).
|
||||
*/
|
||||
export class RemoteDiskLineage {
|
||||
#handler: RemoteHandlerAbstract
|
||||
#vdiDir: string
|
||||
#opts: ResolvedBackupCleanOptions
|
||||
|
||||
// Concrete disk paths discovered in this VDI directory (normalized)
|
||||
#diskPaths: Set<string> = new Set()
|
||||
// Disk paths that exist on disk but failed to open (corrupted)
|
||||
#brokenDiskPaths: Set<string> = new Set()
|
||||
// child path: parent path (from disk header)
|
||||
#parentOf: Map<string, string> = new Map()
|
||||
// parent path: child path
|
||||
#childOf: Map<string, string> = new Map()
|
||||
// Disk paths declared active by their owning archives (accumulated across all referencing archives)
|
||||
#activeDiskPaths: Set<string> = new Set()
|
||||
// Interrupted merges: normalized parent path { stateFilePath, chain }
|
||||
#interruptedMerges: Map<string, { stateFilePath: string; chain?: string[] }> = new Map()
|
||||
|
||||
constructor(handler: RemoteHandlerAbstract, vdiDir: string, opts: ResolvedBackupCleanOptions) {
|
||||
this.#handler = handler
|
||||
this.#vdiDir = vdiDir
|
||||
this.#opts = opts
|
||||
}
|
||||
|
||||
#unregisterDisk(diskPath: string): void {
|
||||
this.#diskPaths.delete(diskPath)
|
||||
const parent = this.#parentOf.get(diskPath)
|
||||
if (parent !== undefined) {
|
||||
this.#parentOf.delete(diskPath)
|
||||
this.#childOf.delete(parent)
|
||||
}
|
||||
const child = this.#childOf.get(diskPath)
|
||||
if (child !== undefined) {
|
||||
this.#childOf.delete(diskPath)
|
||||
this.#parentOf.delete(child)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists disk files and reads their headers to build the parent-child chain.
|
||||
* Also detects interrupted merge state files.
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
const files = await this.#handler.list(this.#vdiDir, { prependDir: true })
|
||||
|
||||
for (const filePath of files) {
|
||||
if (isDisk(filePath)) {
|
||||
this.#diskPaths.add(normalize(filePath))
|
||||
}
|
||||
}
|
||||
|
||||
const uuidToPath = new Map<string, string>()
|
||||
for (const diskPath of this.#diskPaths) {
|
||||
let disk: RemoteDisk | undefined
|
||||
try {
|
||||
disk = await openDisk({ handler: this.#handler, path: diskPath })
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_SUPPORTED' && this.#opts.merge) {
|
||||
throw error
|
||||
}
|
||||
this.#opts.logWarn('failed to open disk', { path: diskPath, error })
|
||||
// Broken disks will be removed by the clean phase
|
||||
// because they are marked as NOT active
|
||||
this.#brokenDiskPaths.add(diskPath)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const uuid = disk.getUuid()
|
||||
// Detect Disks with the same UUIDs
|
||||
// Due to a bug introduced in a1bcd35e2
|
||||
const existingPath = uuidToPath.get(uuid)
|
||||
if (existingPath !== undefined) {
|
||||
this.#opts.logWarn('duplicate disk UUID detected', { uuid, path1: existingPath, path2: diskPath })
|
||||
let existingDisk: RemoteDisk | undefined
|
||||
try {
|
||||
existingDisk = await openDisk({ handler: this.#handler, path: existingPath })
|
||||
if (existingDisk.containsAllDataOf(disk)) {
|
||||
this.#opts.logWarn('should drop duplicate disk, existing is superset', { dropped: diskPath })
|
||||
continue
|
||||
} else if (disk.containsAllDataOf(existingDisk)) {
|
||||
this.#opts.logWarn('should drop old duplicate disk, new is superset', { dropped: existingPath })
|
||||
} else {
|
||||
this.#opts.logWarn('duplicate disks have different content, should first-seen', { dropped: diskPath })
|
||||
continue
|
||||
}
|
||||
} catch (error) {
|
||||
this.#opts.logWarn('failed to open existing duplicate disk, dropping it', { path: existingPath, error })
|
||||
this.#unregisterDisk(existingPath)
|
||||
} finally {
|
||||
await existingDisk?.close()
|
||||
}
|
||||
}
|
||||
uuidToPath.set(uuid, diskPath)
|
||||
|
||||
if (disk.isDifferencing()) {
|
||||
const parentPath = disk.getParentPath()
|
||||
this.#parentOf.set(diskPath, parentPath)
|
||||
if (this.#childOf.has(parentPath)) {
|
||||
const error = new Error('this script does not support multiple children', {
|
||||
cause: {
|
||||
parent: parentPath,
|
||||
child1: this.#childOf.get(parentPath),
|
||||
child2: diskPath,
|
||||
},
|
||||
})
|
||||
throw error
|
||||
}
|
||||
this.#childOf.set(parentPath, diskPath)
|
||||
}
|
||||
} catch (error) {
|
||||
this.#opts.logWarn('failed to read disk parent info', { path: diskPath, error })
|
||||
} finally {
|
||||
await disk.close()
|
||||
}
|
||||
}
|
||||
|
||||
this.#interruptedMerges = await MergeRemoteDisk.findInterruptedMerges(
|
||||
this.#handler,
|
||||
this.#vdiDir,
|
||||
files,
|
||||
uuidToPath,
|
||||
this.#childOf
|
||||
)
|
||||
|
||||
const propagateBroken = (diskPath: string) => {
|
||||
if (!this.#brokenDiskPaths.has(diskPath)) {
|
||||
this.#brokenDiskPaths.add(diskPath)
|
||||
const child = this.#childOf.get(diskPath)
|
||||
if (child !== undefined) {
|
||||
propagateBroken(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [child, parent] of this.#parentOf) {
|
||||
if (!this.#diskPaths.has(parent) || this.#brokenDiskPaths.has(parent)) {
|
||||
this.#opts.logWarn('disk broken: parent missing or broken', { child, parent })
|
||||
propagateBroken(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the disk path exists on disk and opened without errors during init.
|
||||
* Used by VmIncrementalBackupArchive.check() to detect corrupted disks without extra I/O.
|
||||
*/
|
||||
isPathAccessible(diskPath: string): boolean {
|
||||
return this.#diskPaths.has(diskPath) && !this.#brokenDiskPaths.has(diskPath)
|
||||
}
|
||||
|
||||
/** Returns true if the disk exists but failed to open during init. */
|
||||
isBroken(diskPath: string): boolean {
|
||||
return this.#brokenDiskPaths.has(diskPath)
|
||||
}
|
||||
|
||||
/** Verifies that every parent referenced in disk headers exists in the directory. */
|
||||
async check(): Promise<void> {
|
||||
for (const [child, parent] of this.#parentOf) {
|
||||
if (!this.#diskPaths.has(parent)) {
|
||||
this.#opts.logWarn('parent disk is missing', { child, parent })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Declares disk paths as active for this lineage.
|
||||
* Called by each archive that references this lineage after it determines it is complete.
|
||||
* Accumulated across all referencing archives before clean() is called.
|
||||
*/
|
||||
addActiveDiskPath(diskPath: string): void {
|
||||
this.#activeDiskPaths.add(normalize(diskPath))
|
||||
}
|
||||
|
||||
/**
|
||||
* For each orphan disk:
|
||||
* - If it has an active descendant in the chain → build a merge chain and merge (if opts.merge).
|
||||
* - If it has no active descendant → delete it (if opts.remove).
|
||||
*
|
||||
* Also resumes interrupted merges (detected via .merge.json files) when opts.merge is true.
|
||||
* Deletes orphan merge state files (for missing disks) when opts.remove is true.
|
||||
*
|
||||
* Requires activeDiskPaths to be completed to have been called by all referencing archives before this.
|
||||
* @returns Set of deleted disk paths.
|
||||
*/
|
||||
async clean({
|
||||
remove = this.#opts.remove ?? false,
|
||||
merge = this.#opts.merge ?? false,
|
||||
}: { remove?: boolean; merge?: boolean } = {}): Promise<CleanResult> {
|
||||
const orphanDisks = new Set([...this.#diskPaths].filter(p => !this.#activeDiskPaths.has(p)))
|
||||
const toDelete = new Set<string>()
|
||||
const toMerge: { chain: string[]; isResuming: boolean }[] = []
|
||||
const visited = new Set<string>()
|
||||
|
||||
// Walk from an orphan toward its descendants.
|
||||
// Returns a merge chain [orphan, ..., activeAnchor] if an active child exists,
|
||||
// or undefined if the subtree is entirely orphaned (mark for deletion).
|
||||
const getUsedChildChainOrDelete = (diskPath: string): string[] | undefined => {
|
||||
if (visited.has(diskPath)) {
|
||||
return undefined
|
||||
}
|
||||
visited.add(diskPath)
|
||||
|
||||
if (!orphanDisks.has(diskPath)) {
|
||||
// Active disk, anchor of the merge chain
|
||||
return [diskPath]
|
||||
}
|
||||
|
||||
const child = this.#childOf.get(diskPath)
|
||||
if (child !== undefined) {
|
||||
const chain = getUsedChildChainOrDelete(child)
|
||||
if (chain !== undefined) {
|
||||
chain.unshift(diskPath)
|
||||
return chain
|
||||
}
|
||||
}
|
||||
|
||||
toDelete.add(diskPath)
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Process interrupted merges first so their disks are protected from the orphan loop
|
||||
for (const [parentPath, { stateFilePath, chain: stateChain }] of this.#interruptedMerges) {
|
||||
if (!this.#diskPaths.has(parentPath)) {
|
||||
this.#opts.logWarn('orphan merge state', { stateFilePath, missingDisk: parentPath })
|
||||
if (remove) {
|
||||
this.#opts.logInfo('deleting orphan merge state', { stateFilePath })
|
||||
await this.#handler.unlink(stateFilePath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
let chain: string[] | undefined
|
||||
if (stateChain !== undefined) {
|
||||
const existing = stateChain.filter(p => this.#diskPaths.has(p))
|
||||
if (existing.length >= 2) chain = existing
|
||||
}
|
||||
|
||||
if (chain !== undefined) {
|
||||
chain.forEach(p => visited.add(p))
|
||||
toMerge.push({ chain, isResuming: true })
|
||||
}
|
||||
}
|
||||
|
||||
for (const orphan of orphanDisks) {
|
||||
if (!visited.has(orphan)) {
|
||||
const parent = this.#parentOf.get(orphan)
|
||||
if (parent === undefined || !orphanDisks.has(parent)) {
|
||||
const chain = getUsedChildChainOrDelete(orphan)
|
||||
if (chain !== undefined && chain.length > 1) {
|
||||
toMerge.push({ chain, isResuming: false })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (remove) {
|
||||
await asyncEach(
|
||||
Array.from(toDelete),
|
||||
async path => {
|
||||
this.#opts.logInfo('deleting unused disk', { path })
|
||||
try {
|
||||
const disk = await openDisk({ handler: this.#handler, path, ignoreBlockIndexes: true })
|
||||
await disk.unlink({ force: true })
|
||||
} catch (error) {
|
||||
this.#opts.logWarn('failed to delete unused disk', { path, error })
|
||||
}
|
||||
},
|
||||
{ concurrency: DEFAULT_REMOVE_CONCURRENCY }
|
||||
)
|
||||
}
|
||||
|
||||
if (toMerge.length > 0) {
|
||||
this.#opts.logInfo('Disk chain needs merging', { count: toMerge.length })
|
||||
}
|
||||
|
||||
// mergeTargetPath: final size of the disk everything was merged into
|
||||
const mergedSizes = new Map<string, number>()
|
||||
if (merge) {
|
||||
const limitedMergeChain = defaultMergeLimiter(this.#mergeChain.bind(this))
|
||||
const doMerge = async () => {
|
||||
await asyncEach(
|
||||
toMerge,
|
||||
async ({ chain, isResuming }) => {
|
||||
const { finalDiskSize, mergeTargetPath } = await limitedMergeChain(chain, isResuming)
|
||||
mergedSizes.set(mergeTargetPath, (mergedSizes.get(mergeTargetPath) ?? 0) + finalDiskSize)
|
||||
},
|
||||
{ concurrency: this.#opts.mergeConcurrency ?? DEFAULT_MERGE_CONCURRENCY }
|
||||
)
|
||||
}
|
||||
if (toMerge.length !== 0) {
|
||||
await Task.run({ properties: { name: 'merge' } }, doMerge)
|
||||
}
|
||||
}
|
||||
|
||||
await this.#cleanOrphanDataFiles(remove)
|
||||
|
||||
return {
|
||||
removedFiles: Array.from(toDelete),
|
||||
merge: toMerge.length > 0,
|
||||
size: [...mergedSizes.values()].reduce((total, subSize) => total + subSize, 0),
|
||||
mergedSizes,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges ancestors into the active child at the end of the chain.
|
||||
* Returns the final size of the merge target and its path.
|
||||
*/
|
||||
async #mergeChain(chain: string[], isResuming: boolean): Promise<{ finalDiskSize: number; mergeTargetPath: string }> {
|
||||
assert.ok(chain.length >= 2, `look to merge a chain shorter than 2 ${JSON.stringify(chain)}`)
|
||||
const parentPath = chain[0]
|
||||
this.#opts.logInfo('merging disk chain', { chain: [...chain] }) // need a copy here to log full chain
|
||||
// The last disk in the chain is the active one that everything gets merged into
|
||||
const mergeTargetPath = chain.pop()!
|
||||
|
||||
const merger = new MergeRemoteDisk(this.#handler, {
|
||||
logInfo: this.#opts.logInfo,
|
||||
removeUnused: this.#opts.remove,
|
||||
mergeBlockConcurrency: this.#opts.mergeBlockConcurrency,
|
||||
onProgress: this.#opts.onProgress,
|
||||
})
|
||||
let parentDisk: RemoteDisk | undefined
|
||||
let childDisk: RemoteDisk | undefined
|
||||
try {
|
||||
parentDisk = await openDisk({ handler: this.#handler, path: parentPath, force: isResuming })
|
||||
childDisk = await openDiskChain({
|
||||
handler: this.#handler,
|
||||
path: mergeTargetPath,
|
||||
until: parentPath,
|
||||
force: isResuming,
|
||||
})
|
||||
|
||||
const { finalDiskSize } = (await merger.merge(parentDisk, childDisk!))!
|
||||
return { finalDiskSize, mergeTargetPath }
|
||||
} finally {
|
||||
await parentDisk?.close().catch(error => {
|
||||
this.#opts.logWarn('Error closing parent Disk', error)
|
||||
})
|
||||
await childDisk?.close().catch(error => {
|
||||
this.#opts.logWarn('Error closing child Disk', error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates each disk integrity, then scans discovered data subdirs
|
||||
* and deletes any files not associated to a known disk.
|
||||
*/
|
||||
async #cleanOrphanDataFiles(remove: boolean): Promise<void> {
|
||||
const claimedFiles = new Set<string>()
|
||||
for (const diskPath of this.#diskPaths) {
|
||||
const claimed = await Disposable.use(
|
||||
openDisposableDisk({ handler: this.#handler, path: diskPath, ignoreBlockIndexes: true }),
|
||||
async disk => {
|
||||
await disk.clean({ remove, logWarn: this.#opts.logWarn, logInfo: this.#opts.logInfo })
|
||||
return disk.listAssociatedFiles(this.#vdiDir)
|
||||
}
|
||||
)
|
||||
for (const f of claimed) claimedFiles.add(normalize(f))
|
||||
}
|
||||
|
||||
const dataDirs = new Set([...claimedFiles].map(p => dirname(normalize(p))).filter(p => p !== this.#vdiDir))
|
||||
for (const dataDir of dataDirs) {
|
||||
const items = await this.#handler.list(dataDir, { prependDir: true }).catch(() => [] as string[])
|
||||
for (const item of items) {
|
||||
if (!claimedFiles.has(normalize(item))) {
|
||||
this.#opts.logWarn('orphaned data file', { path: item })
|
||||
if (remove) {
|
||||
await this.#handler.unlink(item).catch(async err => {
|
||||
if (err?.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(item)
|
||||
} else {
|
||||
this.#opts.logWarn('failed to delete orphaned data file', { path: item, error: err })
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
|
||||
|
||||
export interface PartialBackupMetadata {
|
||||
mode: 'full' | 'delta'
|
||||
xva?: string
|
||||
vhds?: Record<string, string>
|
||||
vdis?: Record<string, unknown>
|
||||
isVhdDifferencing?: Record<string, boolean>
|
||||
size?: number
|
||||
jobId: string
|
||||
scheduleId: string
|
||||
timestamp: number
|
||||
}
|
||||
export const DEFAULT_MERGE_CONCURRENCY = 1
|
||||
export const DEFAULT_REMOVE_CONCURRENCY = 4
|
||||
|
||||
export interface BackupCleanOptions {
|
||||
fix?: boolean
|
||||
merge?: boolean
|
||||
remove?: boolean
|
||||
mergeBlockConcurrency?: number
|
||||
mergeConcurrency?: number
|
||||
onProgress?: (progress: { total: number; done: number }) => void
|
||||
logInfo?: (message: any, opts?: object) => void
|
||||
logWarn?: (message: any, opts?: object) => void
|
||||
}
|
||||
|
||||
export type ResolvedBackupCleanOptions = BackupCleanOptions & {
|
||||
logInfo: (message: any, opts?: object) => void
|
||||
logWarn: (message: any, opts?: object) => void
|
||||
}
|
||||
|
||||
export interface ArchiveCleanOptions {
|
||||
remove?: boolean
|
||||
merge?: boolean
|
||||
mergedSizes?: Map<string, number>
|
||||
}
|
||||
|
||||
export interface CheckResult {
|
||||
isValid: boolean
|
||||
// Files referenced by metadata but absent on disk
|
||||
missingDisks?: string[]
|
||||
// Disk files that exist but failed to open (corrupted)
|
||||
brokenDisks?: string[]
|
||||
// Files present on disk but not referenced by metadata
|
||||
orphans?: string[]
|
||||
// Files referenced by at least one valid backup
|
||||
linked?: string[]
|
||||
}
|
||||
|
||||
export type CleanResult = {
|
||||
removedFiles: string[]
|
||||
merge?: boolean
|
||||
size?: number
|
||||
mergedSizes?: Map<string, number>
|
||||
}
|
||||
|
||||
export interface VmBackupInterface {
|
||||
handler: RemoteHandlerAbstract
|
||||
// metadataPath: string
|
||||
// metadata: PartialBackupMetadata
|
||||
rootPath: string
|
||||
opts: BackupCleanOptions
|
||||
|
||||
init(): Promise<void>
|
||||
check(): Promise<CheckResult>
|
||||
clean(opts?: ArchiveCleanOptions): Promise<CleanResult>
|
||||
getAssociatedFiles(opts: object): Array<string>
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
|
||||
import { basename, normalize } from '@xen-orchestra/fs/path'
|
||||
import { resolve } from 'node:path'
|
||||
import { VmFullBackupArchive } from './VmFullBackupArchive.mjs'
|
||||
import { VmIncrementalBackupArchive } from './VmIncrementalBackupArchive.mjs'
|
||||
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
|
||||
import {
|
||||
ArchiveCleanOptions,
|
||||
BackupCleanOptions,
|
||||
CheckResult,
|
||||
CleanResult,
|
||||
VmBackupInterface,
|
||||
PartialBackupMetadata,
|
||||
ResolvedBackupCleanOptions,
|
||||
DEFAULT_MERGE_CONCURRENCY,
|
||||
} from './VmBackup.types.mjs'
|
||||
import { cleanOrphanDiskDirs } from '@xen-orchestra/backup-archive/disks'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { promisify } from 'node:util'
|
||||
import zlib from 'node:zlib'
|
||||
|
||||
const gzip = promisify(zlib.gzip)
|
||||
const gunzip = promisify(zlib.gunzip)
|
||||
|
||||
const { info: logInfo, warn: logWarn } = createLogger('xo:backup-archive')
|
||||
|
||||
const FILES_TO_KEEP = ['cache.json.gz', 'vdis']
|
||||
|
||||
export class VmBackupDirectory implements VmBackupInterface {
|
||||
handler: RemoteHandlerAbstract
|
||||
rootPath: string
|
||||
files: Array<string> = new Array()
|
||||
orphans: Set<string> = new Set()
|
||||
backupArchives: Map<string, VmBackupInterface> = new Map()
|
||||
opts: ResolvedBackupCleanOptions
|
||||
|
||||
// Cached result of the last check() call; invalidated by init()
|
||||
#checkResult: (CheckResult & { orphans: string[]; linked: string[] }) | undefined = undefined
|
||||
#uniqueLineages: Map<string, RemoteDiskLineage> | undefined = undefined
|
||||
|
||||
constructor(
|
||||
handler: RemoteHandlerAbstract,
|
||||
vmBackupPath: string,
|
||||
opts: BackupCleanOptions = {
|
||||
fix: true,
|
||||
merge: false,
|
||||
remove: false,
|
||||
logInfo,
|
||||
logWarn,
|
||||
}
|
||||
) {
|
||||
this.handler = handler
|
||||
this.rootPath = vmBackupPath
|
||||
this.opts = {
|
||||
...opts,
|
||||
fix: opts.fix ?? true,
|
||||
merge: opts.merge ?? false,
|
||||
remove: opts.remove ?? false,
|
||||
logInfo: opts.logInfo ?? logInfo,
|
||||
logWarn: opts.logWarn ?? logWarn,
|
||||
}
|
||||
}
|
||||
|
||||
async #readCache(path: string): Promise<Record<string, unknown> | undefined> {
|
||||
try {
|
||||
return JSON.parse((await gunzip(await this.handler.readFile(path))).toString())
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') {
|
||||
logWarn('failed to read cache', { error, path })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #writeCache(path: string, data: Record<string, unknown>): Promise<void> {
|
||||
try {
|
||||
await this.handler.writeFile(path, await gzip(JSON.stringify(data)), { flags: 'w' })
|
||||
} catch (error) {
|
||||
logWarn('failed to write cache', { error, path })
|
||||
}
|
||||
}
|
||||
|
||||
async init() {
|
||||
this.files = (await this.handler.list(this.rootPath, { prependDir: true })).map(file => normalize(file))
|
||||
this.#checkResult = undefined
|
||||
this.#uniqueLineages = new Map()
|
||||
|
||||
for (const fullPath of this.files.filter(path => path.endsWith('.json'))) {
|
||||
let metadata: PartialBackupMetadata | undefined = undefined
|
||||
try {
|
||||
metadata = JSON.parse(await this.handler.readFile(fullPath)) satisfies PartialBackupMetadata
|
||||
} catch (error) {
|
||||
this.opts.logWarn(`Issue loading ${basename(fullPath)}`)
|
||||
}
|
||||
if (metadata !== undefined) {
|
||||
try {
|
||||
const backupArchive = await this.instantiateBackupArchive(fullPath, metadata)
|
||||
this.backupArchives.set(fullPath, backupArchive)
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_SUPPORTED') throw error
|
||||
this.opts.logWarn(`Issue loading ${metadata.xva ?? metadata.vhds}`, { json: fullPath, backup: metadata })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getAssociatedFiles({ prefix = false }) {
|
||||
const files = this.files.filter(file => FILES_TO_KEEP.some(pattern => file.endsWith(pattern)))
|
||||
return prefix ? files : files.map(file => basename(file))
|
||||
}
|
||||
|
||||
async check(): Promise<CheckResult & { orphans: string[]; linked: string[] }> {
|
||||
await this.#checkCacheCount()
|
||||
|
||||
for (const backupArchive of this.backupArchives.values()) {
|
||||
await backupArchive.check()
|
||||
}
|
||||
|
||||
// allUsedFiles is used for root-level orphan detection only.
|
||||
// Active disk paths are accumulated per-lineage by each archive during check() above.
|
||||
const allUsedFiles = new Set<string>([
|
||||
...Array.from(this.backupArchives.values()).flatMap(archive => archive.getAssociatedFiles({ prefix: true })),
|
||||
...this.getAssociatedFiles({ prefix: true }),
|
||||
])
|
||||
|
||||
for (const lineage of this.#uniqueLineages!.values()) {
|
||||
await lineage.check()
|
||||
}
|
||||
|
||||
const orphans = this.files.filter(file => !allUsedFiles.has(file))
|
||||
const linked = Array.from(allUsedFiles)
|
||||
this.#checkResult = { isValid: orphans.length === 0, orphans, linked }
|
||||
return this.#checkResult
|
||||
}
|
||||
|
||||
async clean({
|
||||
remove = this.opts.remove ?? false,
|
||||
merge = this.opts.merge ?? false,
|
||||
}: ArchiveCleanOptions = {}): Promise<CleanResult> {
|
||||
// Use cached check result if available, otherwise run check now
|
||||
const { orphans } = this.#checkResult ?? (await this.check())
|
||||
|
||||
let cacheNeedsRegen = false
|
||||
let someLineageMergedOrShouldBe = false
|
||||
|
||||
// Merge/delete orphan disks in VDI directories covered by archives; collect merged sizes per disk path
|
||||
const allMergedSizes = new Map<string, number>()
|
||||
await asyncEach(
|
||||
Array.from(this.#uniqueLineages!.entries()),
|
||||
async ([_vdiDir, lineage]) => {
|
||||
const { mergedSizes, removedFiles, merge: hasPendingMerge } = await lineage.clean({ remove, merge })
|
||||
if (removedFiles.length > 0) {
|
||||
cacheNeedsRegen = true
|
||||
}
|
||||
if (hasPendingMerge) {
|
||||
someLineageMergedOrShouldBe = true
|
||||
}
|
||||
for (const [diskPath, size] of mergedSizes || []) {
|
||||
allMergedSizes.set(diskPath, (allMergedSizes.get(diskPath) ?? 0) + size)
|
||||
}
|
||||
},
|
||||
{ concurrency: DEFAULT_MERGE_CONCURRENCY }
|
||||
)
|
||||
|
||||
// Delete VDI directories not referenced by any archive
|
||||
const coveredDirs = new Set(this.#uniqueLineages!.keys())
|
||||
await cleanOrphanDiskDirs(this.handler, this.rootPath, coveredDirs, {
|
||||
remove,
|
||||
logWarn: this.opts.logWarn,
|
||||
logInfo: this.opts.logInfo,
|
||||
})
|
||||
|
||||
// Let each archive clean its own files (e.g. remove metadata for incomplete backups)
|
||||
// and update metadata with merged sizes if applicable
|
||||
await asyncEach(
|
||||
Array.from(this.backupArchives.values()),
|
||||
async (archive: VmBackupInterface) => {
|
||||
const { removedFiles } = await archive.clean({ remove, mergedSizes: allMergedSizes })
|
||||
if (removedFiles.length > 0) {
|
||||
cacheNeedsRegen = true
|
||||
}
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
)
|
||||
|
||||
if (allMergedSizes.size > 0 || cacheNeedsRegen) {
|
||||
await this.#regenerateCache()
|
||||
}
|
||||
|
||||
// Delete root-level orphan files (stray xva, checksum, json, etc.), skip folders
|
||||
if (remove) {
|
||||
await asyncEach(
|
||||
orphans,
|
||||
async (orphan: string) => {
|
||||
try {
|
||||
await this.handler.unlink(orphan)
|
||||
} catch (error) {
|
||||
if (error?.code === 'EISDIR') {
|
||||
this.opts.logWarn('orphan is a directory, skipping deletion', { path: orphan })
|
||||
} else if (error?.code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
)
|
||||
}
|
||||
const size = [...allMergedSizes.values()].reduce((total, merged) => total + merged, 0)
|
||||
return { removedFiles: orphans, merge: someLineageMergedOrShouldBe, size: size }
|
||||
}
|
||||
|
||||
async #checkCacheCount(): Promise<void> {
|
||||
const cachePath = `${this.rootPath}/cache.json.gz`
|
||||
const existingCache = await this.#readCache(cachePath)
|
||||
const actual = existingCache === undefined ? 0 : Object.keys(existingCache).length
|
||||
const expected = this.backupArchives.size
|
||||
if (actual !== expected) {
|
||||
this.opts.logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
|
||||
}
|
||||
}
|
||||
|
||||
async #regenerateCache(): Promise<void> {
|
||||
const cachePath = `${this.rootPath}/cache.json.gz`
|
||||
const cache: Record<string, object> = {}
|
||||
for (const [path, archive] of this.backupArchives.entries()) {
|
||||
const a = archive as VmFullBackupArchive | VmIncrementalBackupArchive
|
||||
cache[path] = { _filename: path, id: path, ...a.metadata }
|
||||
}
|
||||
await this.#writeCache(cachePath, cache)
|
||||
}
|
||||
|
||||
async instantiateBackupArchive(metadataPath: string, metadata: PartialBackupMetadata) {
|
||||
let backupArchive: VmBackupInterface
|
||||
try {
|
||||
if (metadata.mode === 'full') {
|
||||
backupArchive = new VmFullBackupArchive(
|
||||
this.handler,
|
||||
this.rootPath,
|
||||
metadataPath,
|
||||
metadata,
|
||||
resolve('/', this.rootPath, metadata.xva!),
|
||||
this.opts
|
||||
)
|
||||
} else if (metadata.mode === 'delta') {
|
||||
const rawDiskPaths = metadata.vhds
|
||||
const diskPaths =
|
||||
rawDiskPaths !== undefined ? Object.values(rawDiskPaths).map(p => resolve('/', this.rootPath, p)) : []
|
||||
backupArchive = new VmIncrementalBackupArchive(
|
||||
this.handler,
|
||||
this.rootPath,
|
||||
metadataPath,
|
||||
metadata,
|
||||
diskPaths,
|
||||
this.opts,
|
||||
this.#uniqueLineages
|
||||
)
|
||||
} else {
|
||||
throw new Error(`Mode ${metadata.mode} not supported`)
|
||||
}
|
||||
} catch (error) {
|
||||
this.opts.logWarn(`Error trying to create backupArchive from ${metadataPath}`, { metadata })
|
||||
throw new Error(`Error trying to create backupArchive from ${metadataPath}`, { cause: error })
|
||||
}
|
||||
await backupArchive.init()
|
||||
return backupArchive
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a fresh instance with the given handler/path/opts, then runs init/check/clean.
|
||||
* The `lock` option is accepted but ignored (locking is the caller's responsibility).
|
||||
*/
|
||||
static async cleanVm(
|
||||
handler: RemoteHandlerAbstract,
|
||||
vmBackupPath: string,
|
||||
opts: BackupCleanOptions & { lock?: boolean } = {}
|
||||
) {
|
||||
const { lock: _lock, ...cleanOpts } = opts
|
||||
const dir = new VmBackupDirectory(handler, vmBackupPath, cleanOpts)
|
||||
await dir.init()
|
||||
await dir.check()
|
||||
const { merge, size } = await dir.clean({ remove: cleanOpts.remove, merge: cleanOpts.merge })
|
||||
return { merge, size }
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
import test from 'node:test'
|
||||
import { strict as assert } from 'node:assert'
|
||||
|
||||
import tmp from 'tmp'
|
||||
import fs from 'fs-extra'
|
||||
import * as uuid from 'uuid'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { pFromCallback } from 'promise-toolbox'
|
||||
import { rimraf } from 'rimraf'
|
||||
// eslint-disable-next-line n/no-missing-import
|
||||
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
|
||||
import tar from 'tar-stream'
|
||||
const { beforeEach, afterEach, describe } = test
|
||||
|
||||
let tempDir, handler, vmBackupDir
|
||||
const vmUuid = 'test-vm-uuid'
|
||||
const rootPath = `xo-vm-backups/${vmUuid}`
|
||||
|
||||
async function createMinimalXva() {
|
||||
const pack = tar.pack()
|
||||
pack.entry({ name: 'ova.xml' }, '<value><struct/></value>')
|
||||
pack.finalize()
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of pack) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
handler = getHandler({ url: `file://${tempDir}` })
|
||||
await handler.sync()
|
||||
await fs.mkdirp(`${tempDir}/${rootPath}`)
|
||||
vmBackupDir = new VmBackupDirectory(handler, rootPath)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rimraf(tempDir)
|
||||
await handler.forget()
|
||||
})
|
||||
|
||||
const uniqueId = () => uuid.v1()
|
||||
|
||||
async function createFullBackupMetadata(name, xvaName) {
|
||||
await handler.writeFile(`${rootPath}/${xvaName}`, await createMinimalXva())
|
||||
const size = await handler.getSize(`${rootPath}/${xvaName}`)
|
||||
const metadata = {
|
||||
mode: 'full',
|
||||
xva: `./${xvaName}`,
|
||||
vm: { uuid: vmUuid },
|
||||
jobId: uniqueId(),
|
||||
timestamp: Date.now(),
|
||||
size,
|
||||
}
|
||||
await handler.writeFile(`${rootPath}/${name}`, JSON.stringify(metadata))
|
||||
return metadata
|
||||
}
|
||||
|
||||
describe('VmBackupDirectory with full backups', { concurrency: 1 }, () => {
|
||||
test('init() loads full backup archives from metadata files', async () => {
|
||||
await createFullBackupMetadata('backup1.json', 'backup1.xva')
|
||||
await createFullBackupMetadata('backup2.json', 'backup2.xva')
|
||||
|
||||
await vmBackupDir.init()
|
||||
|
||||
assert.equal(vmBackupDir.backupArchives.size, 2)
|
||||
})
|
||||
|
||||
test('getAssociatedFiles() returns cache files', async () => {
|
||||
// Create a cache file
|
||||
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache-content')
|
||||
await vmBackupDir.init()
|
||||
|
||||
const filesWithPrefix = vmBackupDir.getAssociatedFiles({ prefix: true })
|
||||
const filesWithoutPrefix = vmBackupDir.getAssociatedFiles({ prefix: false })
|
||||
|
||||
assert.equal(filesWithPrefix.length, 1)
|
||||
assert.ok(filesWithPrefix[0].endsWith('cache.json.gz'))
|
||||
assert.deepEqual(filesWithoutPrefix, ['cache.json.gz'])
|
||||
})
|
||||
|
||||
test('check() identifies orphan files', async () => {
|
||||
// Create a valid backup
|
||||
await createFullBackupMetadata('backup1.json', 'backup1.xva')
|
||||
// Create orphan files (not referenced by any metadata)
|
||||
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
|
||||
await handler.writeFile(`${rootPath}/random-file.txt`, 'random')
|
||||
|
||||
await vmBackupDir.init()
|
||||
const { orphans, linked } = await vmBackupDir.check()
|
||||
|
||||
assert.ok(orphans.some(f => f.includes('orphan.xva')))
|
||||
assert.ok(orphans.some(f => f.includes('random-file.txt')))
|
||||
assert.ok(linked.some(f => f.includes('backup1.json')))
|
||||
assert.ok(linked.some(f => f.includes('backup1.xva')))
|
||||
})
|
||||
|
||||
test('clean() removes orphan files', async () => {
|
||||
await createFullBackupMetadata('backup1.json', 'backup1.xva')
|
||||
await handler.writeFile(`${rootPath}/orphan.xva`, 'orphan-content')
|
||||
|
||||
await vmBackupDir.init()
|
||||
await vmBackupDir.clean({ remove: true })
|
||||
|
||||
const remainingFiles = await handler.list(rootPath)
|
||||
assert.ok(remainingFiles.includes('backup1.json'))
|
||||
assert.ok(remainingFiles.includes('backup1.xva'))
|
||||
assert.ok(!remainingFiles.includes('orphan.xva'))
|
||||
})
|
||||
|
||||
test('clean() preserves cache.json.gz', async () => {
|
||||
await createFullBackupMetadata('backup1.json', 'backup1.xva')
|
||||
await handler.writeFile(`${rootPath}/cache.json.gz`, 'cache')
|
||||
|
||||
await vmBackupDir.init()
|
||||
await vmBackupDir.clean()
|
||||
|
||||
const remainingFiles = await handler.list(rootPath)
|
||||
assert.equal(remainingFiles.length, 3)
|
||||
assert.ok(remainingFiles.includes('cache.json.gz'))
|
||||
})
|
||||
|
||||
test('clean() removes an orphan XVA checksum file (no matching metadata or XVA)', async () => {
|
||||
await handler.writeFile(`${rootPath}/stray.xva.checksum`, 'sha256:deadbeef')
|
||||
|
||||
await vmBackupDir.init()
|
||||
await vmBackupDir.clean({ remove: true })
|
||||
|
||||
const remainingFiles = await handler.list(rootPath)
|
||||
assert.ok(!remainingFiles.includes('stray.xva.checksum'), 'orphan XVA checksum should be deleted')
|
||||
})
|
||||
|
||||
test('clean() keeps an XVA checksum file alongside its valid full backup', async () => {
|
||||
await createFullBackupMetadata('backup1.json', 'backup1.xva')
|
||||
await handler.writeFile(`${rootPath}/backup1.xva.checksum`, 'sha256:abc123')
|
||||
|
||||
await vmBackupDir.init()
|
||||
await vmBackupDir.clean({ remove: true })
|
||||
|
||||
const remainingFiles = await handler.list(rootPath)
|
||||
assert.ok(remainingFiles.includes('backup1.xva'), 'XVA should be kept')
|
||||
assert.ok(vmBackupDir.backupArchives.get(`/${rootPath}/backup1.json`).isValid)
|
||||
assert.ok(remainingFiles.includes('backup1.xva.checksum'), 'checksum should be kept')
|
||||
})
|
||||
})
|
||||
@@ -1,168 +0,0 @@
|
||||
import { basename, normalize } from '@xen-orchestra/fs/path'
|
||||
import assert from 'node:assert'
|
||||
import { FileDescriptor } from '@xen-orchestra/fs'
|
||||
import {
|
||||
ArchiveCleanOptions,
|
||||
CheckResult,
|
||||
CleanResult,
|
||||
VmBackupInterface,
|
||||
PartialBackupMetadata,
|
||||
ResolvedBackupCleanOptions,
|
||||
} from './VmBackup.types.mjs'
|
||||
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
|
||||
|
||||
const COMPRESSED_MAGIC_NUMBERS: Buffer[] = [
|
||||
// https://tools.ietf.org/html/rfc1952.html#page-5
|
||||
Buffer.from('1F8B', 'hex'),
|
||||
|
||||
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
|
||||
Buffer.from('28B52FFD', 'hex'),
|
||||
]
|
||||
|
||||
const MAGIC_NUMBER_MAX_LENGTH: number = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(buf => buf.length))
|
||||
|
||||
const isCompressedFile = async (handler: RemoteHandlerAbstract, fd: FileDescriptor): Promise<boolean> => {
|
||||
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
|
||||
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
|
||||
|
||||
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
|
||||
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const isValidTar = async (handler: RemoteHandlerAbstract, size: number, fd: FileDescriptor): Promise<boolean> => {
|
||||
if (size <= 1024 || size % 512 !== 0) {
|
||||
return false
|
||||
}
|
||||
|
||||
const buf = Buffer.allocUnsafe(1024)
|
||||
assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
|
||||
return buf.every(byte => byte === 0)
|
||||
}
|
||||
|
||||
export async function isValidXva(handler: RemoteHandlerAbstract, filePath: string): Promise<boolean> {
|
||||
// size is longer when encrypted + reading part of an encrypted file is not implemented
|
||||
if (handler.isEncrypted) {
|
||||
return true
|
||||
}
|
||||
|
||||
const fd = await handler.openFile(filePath, 'r')
|
||||
try {
|
||||
const size = await handler.getSize(filePath)
|
||||
if (size < 20) {
|
||||
// neither a valid gzip nor tar
|
||||
return false
|
||||
}
|
||||
|
||||
return (await isCompressedFile(handler, fd))
|
||||
? true // compressed files cannot be validated at this time
|
||||
: await isValidTar(handler, size, fd)
|
||||
} finally {
|
||||
handler.closeFile(fd).catch(noop)
|
||||
}
|
||||
}
|
||||
|
||||
const noop = (): void => {}
|
||||
|
||||
export class VmFullBackupArchive implements VmBackupInterface {
|
||||
handler: RemoteHandlerAbstract
|
||||
rootPath: string
|
||||
xvaPath: string
|
||||
isValid?: boolean
|
||||
missingDisk = false
|
||||
metadataPath: string
|
||||
metadata: PartialBackupMetadata
|
||||
opts: ResolvedBackupCleanOptions
|
||||
|
||||
constructor(
|
||||
handler: RemoteHandlerAbstract,
|
||||
rootPath: string, // xo-vm-backups/<vmUuid>/
|
||||
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
|
||||
metadata: PartialBackupMetadata,
|
||||
xvaPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.xva
|
||||
opts: ResolvedBackupCleanOptions
|
||||
) {
|
||||
this.handler = handler
|
||||
this.rootPath = normalize(rootPath)
|
||||
this.metadataPath = normalize(metadataPath)
|
||||
this.metadata = metadata
|
||||
this.xvaPath = normalize(xvaPath)
|
||||
this.opts = opts
|
||||
}
|
||||
|
||||
async init() {}
|
||||
|
||||
/**
|
||||
* As we are not 100% sure if XVA is valid,
|
||||
* we prefer to warn if it is the case but not remove it.
|
||||
*/
|
||||
async check(): Promise<CheckResult> {
|
||||
if (this.isValid === undefined) {
|
||||
let fileSize = 0
|
||||
let validDisk = false
|
||||
let validSize = true
|
||||
try {
|
||||
fileSize = await this.handler.getSize(this.xvaPath)
|
||||
validDisk = await isValidXva(this.handler, this.xvaPath)
|
||||
} catch (error) {
|
||||
validDisk = false
|
||||
if (error?.code === 'ENOENT') {
|
||||
this.missingDisk = true
|
||||
}
|
||||
this.opts.logWarn('Issue while checking XVA', { error })
|
||||
}
|
||||
try {
|
||||
await this.handler.getSize(`${this.xvaPath}.checksum`)
|
||||
} catch (error) {
|
||||
this.opts.logWarn('Checksum file not valid, not blocking', { error })
|
||||
}
|
||||
if (this.metadata.size !== undefined) {
|
||||
validSize = this.metadata.size === fileSize
|
||||
}
|
||||
this.isValid = fileSize > 0 && validSize && validDisk
|
||||
}
|
||||
if (!this.isValid) {
|
||||
this.opts.logWarn('XVA might be broken', { path: this.xvaPath })
|
||||
}
|
||||
return { isValid: this.isValid }
|
||||
}
|
||||
|
||||
async clean({ remove = this.opts.remove ?? false }: ArchiveCleanOptions = {}): Promise<CleanResult> {
|
||||
if (this.isValid === undefined) {
|
||||
await this.check()
|
||||
}
|
||||
const removedFiles: string[] = []
|
||||
|
||||
if (!this.isValid && this.missingDisk) {
|
||||
removedFiles.push(this.metadataPath, `${this.xvaPath}.checksum`)
|
||||
}
|
||||
|
||||
if (remove) {
|
||||
if (!this.isValid) {
|
||||
if (!this.missingDisk) {
|
||||
this.opts.logWarn(`This files may be corrupted but not yet to be removed`, {
|
||||
files: this.getAssociatedFiles({ prefix: false }),
|
||||
})
|
||||
} else {
|
||||
for (const file of removedFiles) {
|
||||
try {
|
||||
await this.handler.unlink(file)
|
||||
} catch (error) {
|
||||
this.opts.logWarn(`Issue removing ${file}`, { error })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { removedFiles, merge: false }
|
||||
}
|
||||
|
||||
getAssociatedFiles({ prefix = false }): Array<string> {
|
||||
let validFiles = [this.metadataPath, this.xvaPath, `${this.xvaPath}.checksum`]
|
||||
return prefix ? validFiles : validFiles.map(file => basename(file))
|
||||
}
|
||||
}
|
||||
@@ -1,688 +0,0 @@
|
||||
import test from 'node:test'
|
||||
import { strict as assert } from 'node:assert'
|
||||
|
||||
import tmp from 'tmp'
|
||||
import fs from 'fs-extra'
|
||||
import * as uuid from 'uuid'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { pFromCallback } from 'promise-toolbox'
|
||||
// eslint-disable-next-line n/no-missing-import
|
||||
import { VmBackupDirectory } from '../dist/VmBackupDirectory.mjs'
|
||||
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
|
||||
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
|
||||
import { dirname, basename } from 'node:path'
|
||||
import { rimraf } from 'rimraf'
|
||||
|
||||
const { beforeEach, afterEach, describe } = test
|
||||
|
||||
let tempDir, handler, jobId, vdiId, basePath, relativePath, vdiId2, basePath2, relativePath2
|
||||
const rootPath = 'xo-vm-backups/VMUUID/'
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
handler = getHandler({ url: `file://${tempDir}` })
|
||||
await handler.sync()
|
||||
jobId = uniqueId()
|
||||
vdiId = uniqueId()
|
||||
relativePath = `vdis/${jobId}/${vdiId}`
|
||||
basePath = `${rootPath}/${relativePath}`
|
||||
vdiId2 = uniqueId()
|
||||
relativePath2 = `vdis/${jobId}/${vdiId2}`
|
||||
basePath2 = `${rootPath}/${relativePath2}`
|
||||
await fs.mkdirp(`${tempDir}/${basePath}`)
|
||||
await fs.mkdirp(`${tempDir}/${basePath2}`)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rimraf(tempDir)
|
||||
await handler.forget()
|
||||
})
|
||||
|
||||
const uniqueId = () => uuid.v1()
|
||||
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
|
||||
|
||||
async function generateVhd(path, opts = {}) {
|
||||
let vhd
|
||||
|
||||
let dataPath = path
|
||||
if (opts.useAlias) {
|
||||
await handler.mkdir(dirname(path) + '/data/')
|
||||
dataPath = dirname(path) + '/data/' + basename(path)
|
||||
}
|
||||
if (opts.mode === 'directory') {
|
||||
await handler.mkdir(dataPath)
|
||||
vhd = new VhdDirectory(handler, dataPath)
|
||||
} else {
|
||||
const fd = await handler.openFile(dataPath, 'wx')
|
||||
vhd = new VhdFile(handler, fd)
|
||||
}
|
||||
|
||||
vhd.header = { ...VHDHEADER, ...opts.header }
|
||||
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
|
||||
|
||||
if (vhd.header.parentUuid) {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
|
||||
} else {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
|
||||
}
|
||||
|
||||
if (opts.useAlias === true) {
|
||||
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
|
||||
}
|
||||
|
||||
if (opts.blocks) {
|
||||
for (const blockId of opts.blocks) {
|
||||
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
|
||||
}
|
||||
}
|
||||
await vhd.writeBlockAllocationTable()
|
||||
await vhd.writeHeader()
|
||||
await vhd.writeFooter()
|
||||
return vhd
|
||||
}
|
||||
|
||||
test('It remove broken vhd', async () => {
|
||||
// todo also tests a directory and an alias
|
||||
|
||||
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
|
||||
// metadata references a non-existent disk so the lineage is created for this vdiDir
|
||||
// and notReallyAVhd.vhd is treated as an unreferenced orphan
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
|
||||
)
|
||||
assert.equal((await handler.list(basePath)).length, 1)
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
logged += message
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: false, logInfo, logWarn: logInfo })
|
||||
assert.ok(logged.includes('failed to open disk'))
|
||||
// not removed
|
||||
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
|
||||
// really remove it
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
|
||||
assert.deepEqual(await handler.list(basePath), [])
|
||||
})
|
||||
|
||||
test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
// one with a broken parent, should be deleted
|
||||
await generateVhd(`${basePath}/abandoned.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'gone.vhd',
|
||||
parentUuid: uniqueIdBuffer(),
|
||||
},
|
||||
})
|
||||
|
||||
// one orphan, which is a full vhd, no parent : should stay
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan in the metadata : should stay
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUuid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
|
||||
})
|
||||
)
|
||||
// clean
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
logged += message + '\n'
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
|
||||
|
||||
// abandoned has a missing parent, orphan and child have no active backup reference
|
||||
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
|
||||
assert.equal(detectedMissingParent.length, 1)
|
||||
|
||||
// we don't test the file on disk, since they will all be marked as unused and deleted without a metadata.json file
|
||||
})
|
||||
|
||||
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
|
||||
// create a metadata file marking child and orphan as ok
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${relativePath}/orphan.vhd`,
|
||||
`${relativePath}/child.vhd`,
|
||||
// abandoned.json is not here
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
await generateVhd(`${basePath}/abandoned.vhd`)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
|
||||
// a child to the orphan
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUuid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
logged += message + '\n'
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
|
||||
let matched = logged.match(/deleting unused disk/g) || []
|
||||
assert.equal(matched.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// a missing vhd cause clean to remove all vhds
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
|
||||
`${relativePath}/orphan.vhd`,
|
||||
`${relativePath}/child.vhd`,
|
||||
// abandoned.vhd is not here anymore
|
||||
],
|
||||
}),
|
||||
{ flags: 'w' }
|
||||
)
|
||||
logged = ''
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: () => {} })
|
||||
matched = logged.match(/deleting unused disk/g) || []
|
||||
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
|
||||
|
||||
assert.ok(
|
||||
(await handler.list(rootPath)).includes('cache.json.gz'),
|
||||
'cache.json.gz should be regenerated after metadata deletion'
|
||||
)
|
||||
})
|
||||
|
||||
test('it removes metadata and broken vhd when an active backup references a corrupt disk', async () => {
|
||||
// A VHD that exists on disk but is corrupted, openDisk will fail during lineage init
|
||||
await handler.writeFile(`${basePath}/broken.vhd`, 'I AM NOT A VHD')
|
||||
|
||||
// Metadata references the corrupt VHD as if it were a valid active disk
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [`${relativePath}/broken.vhd`],
|
||||
})
|
||||
)
|
||||
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
logged += message
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, {
|
||||
remove: true,
|
||||
logInfo: () => {},
|
||||
logWarn: logInfo,
|
||||
})
|
||||
|
||||
assert.ok(logged.includes('disk check error'), 'should warn that the disk is broken')
|
||||
assert.ok(logged.includes('incremental backup is incomplete'), 'backup should be flagged as incomplete')
|
||||
|
||||
// metadata removed because backup is incomplete
|
||||
assert.ok(!(await handler.list(rootPath)).includes('metadata.json'), 'metadata should be deleted')
|
||||
|
||||
// broken VHD removed via the orphan path in lineage.clean()
|
||||
assert.deepEqual(await handler.list(basePath), [], 'broken VHD should be deleted')
|
||||
})
|
||||
|
||||
test('it preserves all disks when multiple backups reference the same VDI directory', async () => {
|
||||
// Three delta backups share the same VDI directory: snapshot1 ← snapshot2 ← snapshot3.
|
||||
// Each snapshot is the active disk for exactly one backup metadata file.
|
||||
//
|
||||
// Regression guard for a shared-lineage bug: each VmIncrementalBackupArchive creates its
|
||||
// own RemoteDiskLineage for the same vdiDir; only the first archive's lineage reaches
|
||||
// RemoteDiskLineage.clean(). The second and third archives mark their disks as active on
|
||||
// their own (dead) lineage instances, so snapshot2 and snapshot3 appear orphaned and are
|
||||
// deleted — destroying two live backups.
|
||||
|
||||
const snapshot1 = await generateVhd(`${basePath}/snapshot1.vhd`)
|
||||
const snapshot2 = await generateVhd(`${basePath}/snapshot2.vhd`, {
|
||||
header: { parentUnicodeName: 'snapshot1.vhd', parentUuid: snapshot1.footer.uuid },
|
||||
})
|
||||
await generateVhd(`${basePath}/snapshot3.vhd`, {
|
||||
header: { parentUnicodeName: 'snapshot2.vhd', parentUuid: snapshot2.footer.uuid },
|
||||
})
|
||||
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata1.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot1.vhd`] })
|
||||
)
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata2.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot2.vhd`] })
|
||||
)
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata3.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/snapshot3.vhd`] })
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
|
||||
|
||||
const remaining = await handler.list(basePath)
|
||||
assert.ok(remaining.includes('snapshot1.vhd'), 'snapshot1.vhd must survive (referenced by metadata1)')
|
||||
assert.ok(remaining.includes('snapshot2.vhd'), 'snapshot2.vhd must survive (referenced by metadata2)')
|
||||
assert.ok(remaining.includes('snapshot3.vhd'), 'snapshot3.vhd must survive (referenced by metadata3)')
|
||||
assert.equal(remaining.length, 3, 'no extra files should remain')
|
||||
|
||||
// All three metadata files must also survive (all backups are complete)
|
||||
const rootFiles = await handler.list(rootPath)
|
||||
assert.ok(rootFiles.includes('metadata1.json'), 'metadata1.json must survive')
|
||||
assert.ok(rootFiles.includes('metadata2.json'), 'metadata2.json must survive')
|
||||
assert.ok(rootFiles.includes('metadata3.json'), 'metadata3.json must survive')
|
||||
})
|
||||
|
||||
test('it handles multiple VDI directories independently', async () => {
|
||||
// vdiDir1: diskA1 (base) <- diskA2, plus an orphan
|
||||
const diskA1 = await generateVhd(`${basePath}/diskA1.vhd`)
|
||||
await generateVhd(`${basePath}/diskA2.vhd`, {
|
||||
header: { parentUnicodeName: 'diskA1.vhd', parentUuid: diskA1.footer.uuid },
|
||||
})
|
||||
await generateVhd(`${basePath}/orphanA.vhd`)
|
||||
|
||||
// vdiDir2: diskB1 (base) <- diskB2, plus an orphan
|
||||
const diskB1 = await generateVhd(`${basePath2}/diskB1.vhd`)
|
||||
await generateVhd(`${basePath2}/diskB2.vhd`, {
|
||||
header: { parentUnicodeName: 'diskB1.vhd', parentUuid: diskB1.footer.uuid },
|
||||
})
|
||||
await generateVhd(`${basePath2}/orphanB.vhd`)
|
||||
|
||||
// archive1 covers both base disks, archive2 covers both child disks
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata1.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA1.vhd`, `${relativePath2}/diskB1.vhd`] })
|
||||
)
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata2.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/diskA2.vhd`, `${relativePath2}/diskB2.vhd`] })
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
|
||||
|
||||
const remaining1 = await handler.list(basePath)
|
||||
assert.ok(remaining1.includes('diskA1.vhd'), 'diskA1.vhd must survive (metadata1)')
|
||||
assert.ok(remaining1.includes('diskA2.vhd'), 'diskA2.vhd must survive (metadata2)')
|
||||
assert.ok(!remaining1.includes('orphanA.vhd'), 'orphanA.vhd must be deleted')
|
||||
|
||||
const remaining2 = await handler.list(basePath2)
|
||||
assert.ok(remaining2.includes('diskB1.vhd'), 'diskB1.vhd must survive (metadata1)')
|
||||
assert.ok(remaining2.includes('diskB2.vhd'), 'diskB2.vhd must survive (metadata2)')
|
||||
assert.ok(!remaining2.includes('orphanB.vhd'), 'orphanB.vhd must be deleted')
|
||||
})
|
||||
|
||||
test('it merges delta of non destroyed chain', async () => {
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: 12000, // a size too small
|
||||
vhds: [
|
||||
`${relativePath}/grandchild.vhd`, // grand child should not be merged
|
||||
`${relativePath}/child.vhd`,
|
||||
// orphan is not here, he should be merged in child
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUuid: orphan.footer.uuid,
|
||||
},
|
||||
blocks: [0, 1],
|
||||
})
|
||||
// a grand child
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child.vhd',
|
||||
parentUuid: child.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
let logged = []
|
||||
const logInfo = message => {
|
||||
logged.push(message)
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logInfo, logWarn: logInfo })
|
||||
assert.equal(logged[0], `unexpected number of entries in backup cache`)
|
||||
|
||||
logged = []
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
|
||||
assert.equal(logged.includes(`merging disk chain`), true)
|
||||
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
// only check deletion
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
assert.equal(remainingVhds.length, 2)
|
||||
assert.equal(remainingVhds.includes('child.vhd'), true)
|
||||
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
assert.equal(metadata.size, 4299776, 'metadata.size should be updated after merge')
|
||||
})
|
||||
|
||||
test('it merges a chain of multiple consecutive orphan ancestors in one pass', async () => {
|
||||
// ancestor -> child -> grandchild
|
||||
// only grandchild is referenced in metadata: both ancestor and child are orphans
|
||||
// they must be merged as a single 3-disk chain, not processed separately
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: 12000,
|
||||
vhds: [`${relativePath}/grandchild.vhd`],
|
||||
})
|
||||
)
|
||||
|
||||
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
|
||||
})
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
header: { parentUnicodeName: 'child.vhd', parentUuid: child.footer.uuid },
|
||||
})
|
||||
|
||||
const mergeChains = []
|
||||
const logInfo = (message, opts) => {
|
||||
if (message === 'merging disk chain') mergeChains.push(opts.chain)
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logInfo, logWarn: () => {} })
|
||||
|
||||
// all three disks must be handled in exactly one merge, not two separate operations
|
||||
assert.equal(mergeChains.length, 1, 'exactly one merge should occur for the full chain')
|
||||
assert.equal(mergeChains[0].length, 3, 'merge chain must include ancestor, child and grandchild')
|
||||
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
assert.equal(remainingVhds.includes('grandchild.vhd'), true)
|
||||
assert.equal(remainingVhds.includes('ancestor.vhd'), false, 'ancestor should have been merged, not deleted directly')
|
||||
assert.equal(remainingVhds.includes('child.vhd'), false)
|
||||
})
|
||||
|
||||
test('it resumes an interrupted merge with chain field', async () => {
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/child.vhd`] })
|
||||
)
|
||||
|
||||
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1, 2, 3, 4] })
|
||||
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
|
||||
header: { parentUnicodeName: 'ancestor.vhd', parentUuid: ancestor.footer.uuid },
|
||||
blocks: [5, 6],
|
||||
})
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: { parentUnicodeName: 'intermediate.vhd', parentUuid: intermediate.footer.uuid },
|
||||
blocks: [7, 8],
|
||||
})
|
||||
|
||||
await handler.writeFile(
|
||||
`${basePath}/.ancestor.vhd.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: { uuid: '0' },
|
||||
child: { uuid: '0' },
|
||||
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
|
||||
currentBlock: 3,
|
||||
mergedDataSize: 3 * 2 * 1024 * 1024,
|
||||
step: 'mergeBlocks',
|
||||
diskSize: 0,
|
||||
})
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
|
||||
|
||||
const files = await handler.list(basePath)
|
||||
assert.ok(!files.some(f => f.endsWith('.merge.json')), 'merge state file should be deleted after successful resume')
|
||||
assert.ok(!files.includes('ancestor.vhd'), 'ancestor should be removed after merge')
|
||||
assert.ok(!files.includes('intermediate.vhd'), 'intermediate should be removed after merge')
|
||||
assert.ok(files.includes('child.vhd'), 'child (merge target) should survive')
|
||||
})
|
||||
|
||||
test('it resumes an interrupted merge without chain field', async () => {
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: 209920,
|
||||
vhds: [`${relativePath}/orphan.vhd`, `${relativePath}/child.vhd`],
|
||||
})
|
||||
)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUuid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
// a merge in progress file
|
||||
await handler.writeFile(
|
||||
`${basePath}/.orphan.vhd.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: orphan.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: child.header.checksum,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
|
||||
// only check deletion
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
assert.equal(remainingVhds.length, 1)
|
||||
assert.equal(remainingVhds.includes('child.vhd'), true)
|
||||
})
|
||||
|
||||
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
|
||||
// the message and resulting files should be identical to the output with vhd files which is tested independently
|
||||
describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
for (const useAlias of [true, false]) {
|
||||
for (const vhdMode of ['file', 'directory']) {
|
||||
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
|
||||
// a broken VHD
|
||||
if (useAlias) {
|
||||
await handler.mkdir(basePath + '/data')
|
||||
}
|
||||
|
||||
const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
|
||||
|
||||
if (vhdMode === 'directory') {
|
||||
await handler.mkdir(brokenVhdDataPath)
|
||||
} else {
|
||||
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
|
||||
}
|
||||
if (useAlias) {
|
||||
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
|
||||
}
|
||||
|
||||
// a vhd non referenced in metadata
|
||||
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
|
||||
// an abandoned delta vhd without its parent
|
||||
await generateVhd(`${basePath}/abandoned.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'gone.vhd',
|
||||
parentUuid: uniqueIdBuffer(),
|
||||
},
|
||||
})
|
||||
|
||||
// an ancestor of a vhd present in metadata
|
||||
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
blocks: [1, 3],
|
||||
})
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUuid: ancestor.footer.uuid,
|
||||
},
|
||||
blocks: [1, 2],
|
||||
})
|
||||
// a grand child vhd in metadata
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUuid: child.footer.uuid,
|
||||
},
|
||||
blocks: [2, 3],
|
||||
})
|
||||
|
||||
// an older parent that was merging in clean
|
||||
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
})
|
||||
// a clean vhd in metadata
|
||||
const clean = await generateVhd(`${basePath}/clean.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUuid: cleanAncestor.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
await handler.writeFile(
|
||||
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: cleanAncestor.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: clean.header.checksum,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
// the metadata file
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${relativePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
|
||||
`${relativePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
|
||||
`${relativePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
|
||||
],
|
||||
})
|
||||
)
|
||||
if (!useAlias && vhdMode === 'directory') {
|
||||
try {
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
|
||||
} catch (err) {
|
||||
assert.strictEqual(
|
||||
err.code,
|
||||
'NOT_SUPPORTED',
|
||||
'Merging directory without alias should raise a not supported error'
|
||||
)
|
||||
return
|
||||
}
|
||||
assert.strictEqual(true, false, 'Merging directory without alias should raise an error')
|
||||
}
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, merge: true, logWarn: () => {} })
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
assert.deepEqual(metadata.size, 6501888)
|
||||
// cache.json.gz must be regenerated when metadata changes after a merge
|
||||
assert.ok(
|
||||
(await handler.list(rootPath)).includes('cache.json.gz'),
|
||||
'cache.json.gz should be regenerated after a merge'
|
||||
)
|
||||
|
||||
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
// grand child and clean vhd should not have changed
|
||||
const survivors = await handler.list(basePath)
|
||||
if (useAlias) {
|
||||
const dataSurvivors = await handler.list(basePath + '/data')
|
||||
// the goal of the alias : do not move a full folder
|
||||
assert.equal(dataSurvivors.includes('ancestor.vhd'), true)
|
||||
assert.equal(dataSurvivors.includes('grandchild.vhd'), true)
|
||||
assert.equal(dataSurvivors.includes('cleanAncestor.vhd'), true)
|
||||
assert.equal(survivors.includes('clean.vhd.alias.vhd'), true)
|
||||
assert.equal(survivors.includes('child.vhd.alias.vhd'), true)
|
||||
assert.equal(survivors.includes('grandchild.vhd.alias.vhd'), true)
|
||||
assert.equal(survivors.length, 4) // the 3 ok + data
|
||||
assert.equal(dataSurvivors.length, 3)
|
||||
} else {
|
||||
assert.equal(survivors.includes('clean.vhd'), true)
|
||||
assert.equal(survivors.includes('child.vhd'), true)
|
||||
assert.equal(survivors.includes('grandchild.vhd'), true)
|
||||
assert.equal(survivors.length, 3)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
test('it cleans orphan merge states ', async () => {
|
||||
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
|
||||
// metadata is needed so the lineage is created for basePath (otherwise cleanOrphanDiskDirs deletes the whole dir)
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {} })
|
||||
|
||||
assert.deepEqual(await handler.list(basePath), [])
|
||||
})
|
||||
|
||||
test('check all types of aliases, corrupted, missing or not', async () => {
|
||||
// valid alias pointing to an existing data file
|
||||
await handler.mkdir(`${basePath}/data`)
|
||||
await generateVhd(`${basePath}/data/ok.vhd`)
|
||||
await VhdAbstract.createAlias(handler, `${basePath}/ok.alias.vhd`, `${basePath}/data/ok.vhd`)
|
||||
|
||||
// broken alias pointing to a missing data file
|
||||
await VhdAbstract.createAlias(handler, `${basePath}/missingData.alias.vhd`, `${basePath}/data/nonexistent.vhd`)
|
||||
|
||||
// alias pointing to an existing but corrupt data file
|
||||
await handler.writeFile(`${basePath}/data/corrupt.vhd`, 'I AM NOT A VHD')
|
||||
await VhdAbstract.createAlias(handler, `${basePath}/corrupt.alias.vhd`, `${basePath}/data/corrupt.vhd`)
|
||||
|
||||
// data file without alias
|
||||
await generateVhd(`${basePath}/data/missingalias.vhd`)
|
||||
|
||||
// only the valid alias is referenced by a complete backup
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/ok.alias.vhd`] })
|
||||
)
|
||||
|
||||
await VmBackupDirectory.cleanVm(handler, rootPath, { remove: true, logWarn: () => {}, logInfo: () => {} })
|
||||
|
||||
// only ok.alias.vhd has survived
|
||||
const aliases = (await handler.list(basePath)).filter(f => f.endsWith('.vhd'))
|
||||
assert.equal(aliases.length, 1)
|
||||
assert.equal(aliases[0], 'ok.alias.vhd')
|
||||
|
||||
// corrupt.vhd and missingalias.vhd must be gone
|
||||
const data = await handler.list(`${basePath}/data`)
|
||||
assert.equal(data.length, 1)
|
||||
assert.equal(data[0], 'ok.vhd')
|
||||
})
|
||||
@@ -1,174 +0,0 @@
|
||||
import {
|
||||
ArchiveCleanOptions,
|
||||
CheckResult,
|
||||
CleanResult,
|
||||
ResolvedBackupCleanOptions,
|
||||
VmBackupInterface,
|
||||
PartialBackupMetadata,
|
||||
} from './VmBackup.types.mjs'
|
||||
import { RemoteHandlerAbstract } from '@xen-orchestra/fs'
|
||||
import { basename, dirname, normalize } from '@xen-orchestra/fs/path'
|
||||
import { RemoteDiskLineage } from './RemoteDiskLineage.mjs'
|
||||
|
||||
export class VmIncrementalBackupArchive implements VmBackupInterface {
|
||||
handler: RemoteHandlerAbstract
|
||||
metadataPath: string
|
||||
metadata: PartialBackupMetadata
|
||||
readonly diskPaths: Array<string>
|
||||
rootPath: string
|
||||
opts: ResolvedBackupCleanOptions
|
||||
diskLineages: Map<string, RemoteDiskLineage> = new Map()
|
||||
|
||||
#isChecked = false
|
||||
#isComplete = false
|
||||
|
||||
get isChecked(): boolean {
|
||||
return this.#isChecked
|
||||
}
|
||||
|
||||
get isComplete(): boolean {
|
||||
return this.#isComplete
|
||||
}
|
||||
|
||||
constructor(
|
||||
handler: RemoteHandlerAbstract,
|
||||
rootPath: string, // xo-vm-backups/<vmUuid>/
|
||||
metadataPath: string, // xo-vm-backups/<vmUuid>/<timestamp>_<scheduleId>.json
|
||||
metadata: PartialBackupMetadata,
|
||||
diskPaths: Array<string>, // xo-vm-backups/<vmUuid>/vdis/<jobId>/<vdiUuid>/<snapshotUuid>.alias.vhd (one per VDI)
|
||||
opts: ResolvedBackupCleanOptions,
|
||||
lineageRegistry?: Map<string, RemoteDiskLineage>
|
||||
) {
|
||||
this.handler = handler
|
||||
this.rootPath = normalize(rootPath)
|
||||
this.metadataPath = normalize(metadataPath)
|
||||
this.metadata = metadata
|
||||
this.diskPaths = diskPaths.map(path => normalize(path))
|
||||
this.opts = opts
|
||||
if (lineageRegistry !== undefined) {
|
||||
this.diskLineages = lineageRegistry
|
||||
}
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
// Build one RemoteDiskLineage per VDI directory derived from diskPaths
|
||||
try {
|
||||
for (const diskPath of this.diskPaths) {
|
||||
const vdiDir = dirname(diskPath)
|
||||
if (!this.diskLineages.has(vdiDir)) {
|
||||
const lineage = new RemoteDiskLineage(this.handler, vdiDir, this.opts)
|
||||
await lineage.init()
|
||||
this.diskLineages.set(vdiDir, lineage)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code === 'NOT_SUPPORTED') throw error
|
||||
this.opts.logWarn('failed to scan VDI directories', { error })
|
||||
}
|
||||
}
|
||||
|
||||
async check(): Promise<CheckResult> {
|
||||
const missingDisks: string[] = []
|
||||
const brokenDisks: string[] = []
|
||||
|
||||
if (this.diskPaths.length === 0) {
|
||||
this.opts.logWarn('incremental backup has no disk paths', { metadataPath: this.metadataPath })
|
||||
this.#isComplete = false
|
||||
} else {
|
||||
for (const diskPath of this.diskPaths) {
|
||||
const lineage = this.diskLineages.get(dirname(diskPath))
|
||||
if (lineage === undefined || !lineage.isPathAccessible(diskPath)) {
|
||||
if (lineage?.isBroken(diskPath)) {
|
||||
this.opts.logWarn('disk check error', { path: diskPath })
|
||||
brokenDisks.push(diskPath)
|
||||
} else {
|
||||
this.opts.logWarn('disk file missing', { path: diskPath })
|
||||
missingDisks.push(diskPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.#isComplete = missingDisks.length === 0 && brokenDisks.length === 0
|
||||
if (!this.#isComplete) {
|
||||
this.opts.logWarn('incremental backup is incomplete', {
|
||||
metadataPath: this.metadataPath,
|
||||
missingDisks,
|
||||
brokenDisks,
|
||||
})
|
||||
}
|
||||
}
|
||||
if (this.#isComplete) {
|
||||
for (const diskPath of this.diskPaths) {
|
||||
this.diskLineages.get(dirname(diskPath))?.addActiveDiskPath(diskPath)
|
||||
}
|
||||
}
|
||||
|
||||
this.#isChecked = true
|
||||
return { isValid: this.#isComplete, missingDisks, brokenDisks }
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the metadata file if the backup is incomplete (missing disks).
|
||||
* If mergedSizes is provided, updates metadata with the total merged size for this archive's disks.
|
||||
* Actual disk merge/deletion is handled by RemoteDiskLineage.
|
||||
*/
|
||||
async clean({ remove = this.opts.remove ?? false, mergedSizes }: ArchiveCleanOptions = {}): Promise<CleanResult> {
|
||||
if (!this.#isChecked) {
|
||||
await this.check()
|
||||
}
|
||||
|
||||
const removedFiles: string[] = []
|
||||
if (!this.#isComplete) {
|
||||
removedFiles.push(this.metadataPath)
|
||||
}
|
||||
|
||||
if (remove) {
|
||||
for (const file of removedFiles) {
|
||||
try {
|
||||
await this.handler.unlink(file)
|
||||
} catch (error) {
|
||||
this.opts.logWarn(`Issue removing ${file}`, { error })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mergedSize = 0
|
||||
if (mergedSizes !== undefined) {
|
||||
for (const diskPath of this.diskPaths) {
|
||||
mergedSize += mergedSizes.get(diskPath) ?? 0
|
||||
}
|
||||
if (mergedSize > 0) {
|
||||
await this.updateMetadata(mergedSize)
|
||||
}
|
||||
}
|
||||
|
||||
return { removedFiles }
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates metadata.size after a merge and writes the updated metadata file to disk.
|
||||
* Also resets isVhdDifferencing so all VDIs are marked as full (non-differencing).
|
||||
*/
|
||||
async updateMetadata(mergedSize: number): Promise<void> {
|
||||
this.metadata.size = mergedSize
|
||||
if (this.metadata.vdis !== undefined) {
|
||||
this.metadata.isVhdDifferencing = {}
|
||||
for (const id of Object.keys(this.metadata.vdis)) {
|
||||
this.metadata.isVhdDifferencing[id] = false
|
||||
}
|
||||
}
|
||||
await this.handler.writeFile(this.metadataPath, JSON.stringify(this.metadata), { flags: 'w' })
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the metadata file path and active disk paths if the backup is complete, empty otherwise.
|
||||
* Must be called after check().
|
||||
*/
|
||||
getAssociatedFiles({ prefix = false }): Array<string> {
|
||||
if (!this.#isComplete) {
|
||||
return []
|
||||
}
|
||||
const files = [this.metadataPath, ...this.diskPaths]
|
||||
return prefix ? files : files.map(file => basename(file))
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
export { RemoteDisk } from './RemoteDisk.mjs'
|
||||
export { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
export { openDiskChain } from './openDiskChain.mjs'
|
||||
export { MergeRemoteDisk } from './MergeRemoteDisk.mjs'
|
||||
|
||||
const DISK_EXTENSIONS = ['.vhd']
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
* @typedef {import('./MergeRemoteDisk.mjs').MergeState} MergeState
|
||||
*/
|
||||
|
||||
export function isDisk(path) {
|
||||
return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {RemoteHandlerAbstract} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {boolean} [params.force]
|
||||
* @param {boolean} [params.ignoreBlockIndexes]
|
||||
* @returns {Promise<Disposable<RemoteDisk>>}
|
||||
*/
|
||||
export async function openDisposableDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
if (ignoreBlockIndexes) {
|
||||
// Best-effort init: return the disk even if init fails so that callers
|
||||
// can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
|
||||
await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
|
||||
} else {
|
||||
await disk.init({ force })
|
||||
}
|
||||
return {
|
||||
value: disk,
|
||||
dispose: () => disk.close(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {RemoteHandlerAbstract} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {boolean} [params.force]
|
||||
* @param {boolean} [params.ignoreBlockIndexes]
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
export async function openDisk({ handler, path, force = false, ignoreBlockIndexes = false }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
if (ignoreBlockIndexes) {
|
||||
// Best-effort init: return the disk even if init fails so that callers
|
||||
// can still use operations that work without a fully-opened VHD (e.g. unlink, checkAlias).
|
||||
await disk.init({ force, ignoreBlockIndexes }).catch(() => {})
|
||||
} else {
|
||||
await disk.init({ force })
|
||||
}
|
||||
return disk
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes VDI directories under `vmDir/vdis/<jobId>/<vdiId>/` that are not in coveredDirs.
|
||||
* Used to clean up disk directories that no backup metadata references at all.
|
||||
*
|
||||
* @param {RemoteHandlerAbstract} handler
|
||||
* @param {string} vmDir - root of the VM backup directory (xo-vm-backups/<vmUuid>)
|
||||
* @param {Set<string>} coveredDirs - normalized vdiDir paths already managed by a lineage
|
||||
* @param {{ remove?: boolean, logWarn?: Function, logInfo?: Function }} opts
|
||||
*/
|
||||
export async function cleanOrphanDiskDirs(
|
||||
handler,
|
||||
vmDir,
|
||||
coveredDirs,
|
||||
{ remove = false, logWarn = () => {}, logInfo = () => {} } = {}
|
||||
) {
|
||||
const vdisPath = `${vmDir}/vdis`
|
||||
let jobDirs
|
||||
try {
|
||||
jobDirs = await handler.list(vdisPath, { prependDir: true })
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
for (const jobDir of jobDirs) {
|
||||
let vdiDirs
|
||||
try {
|
||||
vdiDirs = await handler.list(jobDir, { prependDir: true })
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const vdiDir of vdiDirs) {
|
||||
if (!coveredDirs.has(vdiDir)) {
|
||||
logWarn('orphan VDI directory, not referenced by any backup', { path: vdiDir })
|
||||
if (remove) {
|
||||
logInfo('deleting orphan VDI directory', { path: vdiDir })
|
||||
try {
|
||||
await handler.rmtree(vdiDir)
|
||||
} catch (error) {
|
||||
logWarn('failed to delete orphan VDI directory', { path: vdiDir, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// @ts-check
|
||||
/**
|
||||
*
|
||||
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
import { defer } from 'golike-defer'
|
||||
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
|
||||
|
||||
/**
|
||||
* @param {any} $defer
|
||||
* @param {Object} params
|
||||
* @param {RemoteHandlerAbstract} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {string | undefined} params.until
|
||||
* @param {boolean} [params.force]
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
async function _openDiskChain($defer, { handler, path, until, force = false }) {
|
||||
/**
|
||||
* @type {RemoteVhdDisk}
|
||||
*/
|
||||
let disk
|
||||
/**
|
||||
* @type {Array<RemoteVhdDisk>}
|
||||
*/
|
||||
const disks = []
|
||||
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
|
||||
await disk.init({ force })
|
||||
disks.push(disk)
|
||||
let foundRootDisk = until === undefined
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent()
|
||||
if (disk.getPath() === until) {
|
||||
foundRootDisk = true
|
||||
await disk.close()
|
||||
break
|
||||
}
|
||||
disks.unshift(disk)
|
||||
}
|
||||
if (!foundRootDisk) {
|
||||
throw new Error(`trying to open chain from ${path} to ${until}, root disk not found`)
|
||||
}
|
||||
// the root disk
|
||||
return new RemoteVhdDiskChain({ disks })
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {(params: { handler: RemoteHandlerAbstract, path: string, until?: string, force?: boolean }) => Promise<RemoteDisk>}
|
||||
*/
|
||||
export const openDiskChain = defer(_openDiskChain)
|
||||
@@ -1 +0,0 @@
|
||||
export { VmBackupDirectory } from './VmBackupDirectory.mjs'
|
||||
@@ -1,92 +0,0 @@
|
||||
// a valid footer of a 2
|
||||
export const VHDFOOTER = {
|
||||
cookie: 'conectix',
|
||||
features: 2,
|
||||
fileFormatVersion: 65536,
|
||||
dataOffset: 512,
|
||||
timestamp: 0,
|
||||
creatorApplication: 'caml',
|
||||
creatorVersion: 1,
|
||||
creatorHostOs: 0,
|
||||
originalSize: 53687091200,
|
||||
currentSize: 53687091200,
|
||||
diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
|
||||
diskType: 3,
|
||||
checksum: 4294962945,
|
||||
uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
|
||||
saved: '',
|
||||
hidden: '',
|
||||
reserved: '',
|
||||
}
|
||||
export const VHDHEADER = {
|
||||
cookie: 'cxsparse',
|
||||
dataOffset: undefined,
|
||||
tableOffset: 2048,
|
||||
headerVersion: 65536,
|
||||
maxTableEntries: 25600,
|
||||
blockSize: 2097152,
|
||||
checksum: 4294964241,
|
||||
parentUuid: null,
|
||||
parentTimestamp: 0,
|
||||
reserved1: 0,
|
||||
parentUnicodeName: '',
|
||||
parentLocatorEntry: [
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
],
|
||||
reserved2: '',
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"declaration": true,
|
||||
"strict": true,
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"target": "ES2022",
|
||||
"rootDir": "./src",
|
||||
"skipLibCheck": true,
|
||||
"noImplicitAny": false,
|
||||
"allowJs": true,
|
||||
"useUnknownInCatchVariables": false,
|
||||
"types": ["node"] // Only use types from Node.js and packages that are explicitly imported
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -19,7 +19,7 @@ async function cleanFiles(vmDirs, { fix, merge, remove }) {
|
||||
try {
|
||||
nbStarted++
|
||||
await new RemoteAdapter(handler).cleanVm(basename(vmDir), {
|
||||
fix,
|
||||
fixMetadata: fix,
|
||||
remove,
|
||||
merge,
|
||||
logInfo: (...args) => console.log(...args),
|
||||
@@ -50,7 +50,7 @@ async function cleanRemote(url, { fix, merge, remove }) {
|
||||
try {
|
||||
nbStarted++
|
||||
await remoteAdapter.cleanVm(join('xo-vm-backups', vmUuid), {
|
||||
fix,
|
||||
fixMetadata: fix,
|
||||
remove,
|
||||
merge,
|
||||
logInfo: (...args) => console.log(...args),
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/backups": "^0.72.1",
|
||||
"@xen-orchestra/backups": "^0.72.0",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"filenamify": "^6.0.0",
|
||||
"getopts": "^2.2.5",
|
||||
|
||||
@@ -10,7 +10,7 @@ import { dirname, join } from 'node:path'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
import { defer } from 'golike-defer'
|
||||
import { NegativeDisk } from '@xen-orchestra/disk-transform'
|
||||
import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { resetVmOtherConfig } from './_otherConfig.mjs'
|
||||
|
||||
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
|
||||
|
||||
@@ -22,7 +22,7 @@ import * as tar from 'tar'
|
||||
import zlib from 'zlib'
|
||||
|
||||
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
|
||||
import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
|
||||
import { cleanVm } from './_cleanVm.mjs'
|
||||
import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { getTmpDir } from './_getTmpDir.mjs'
|
||||
import { isMetadataFile } from './_backupType.mjs'
|
||||
@@ -31,7 +31,8 @@ import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from '
|
||||
import { lvs, pvs } from './_lvm.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
|
||||
import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { ReadAhead } from '@xen-orchestra/disk-transform'
|
||||
|
||||
@@ -701,7 +702,7 @@ export class RemoteAdapter {
|
||||
const handler = this._handler
|
||||
|
||||
if (this.useVhdDirectory()) {
|
||||
return await writeToVhdDirectory({
|
||||
await writeToVhdDirectory({
|
||||
disk,
|
||||
target: {
|
||||
handler,
|
||||
@@ -713,9 +714,8 @@ export class RemoteAdapter {
|
||||
})
|
||||
} else {
|
||||
const stream = await toVhdStream(disk)
|
||||
const size = await this.outputStream(path, stream, { validator, checksum: false })
|
||||
await this.outputStream(path, stream, { validator, checksum: false })
|
||||
await validator(path)
|
||||
return size
|
||||
}
|
||||
}
|
||||
|
||||
@@ -878,14 +878,11 @@ export class RemoteAdapter {
|
||||
}
|
||||
|
||||
Object.assign(RemoteAdapter.prototype, {
|
||||
cleanVm(vmBackupPath, opts = {}) {
|
||||
const { lock = true, ...cleanOpts } = opts
|
||||
cleanVm(vmDir, { lock = true } = {}) {
|
||||
if (lock) {
|
||||
return Disposable.use(this._handler.lock(vmBackupPath), () => {
|
||||
return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
|
||||
})
|
||||
return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
|
||||
} else {
|
||||
return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
|
||||
return cleanVm.apply(this, arguments)
|
||||
}
|
||||
},
|
||||
isValidXva,
|
||||
|
||||
@@ -9,6 +9,7 @@ import { pFromCallback } from 'promise-toolbox'
|
||||
import { RemoteAdapter } from './RemoteAdapter.mjs'
|
||||
import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
|
||||
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
|
||||
import { checkAliases } from './_cleanVm.mjs'
|
||||
import { dirname, basename } from 'node:path'
|
||||
import { rimraf } from 'rimraf'
|
||||
|
||||
@@ -81,17 +82,13 @@ test('It remove broken vhd', async () => {
|
||||
// todo also tests a directory and an alias
|
||||
|
||||
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/notReallyAVhd.vhd`] })
|
||||
)
|
||||
assert.equal((await handler.list(basePath)).length, 1)
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
logged += message
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
|
||||
assert.ok(logged.includes('disk check error'), `expected 'disk check error' in: ${logged}`)
|
||||
assert.equal(logged, `VHD check error`)
|
||||
// not removed
|
||||
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
|
||||
// really remove it
|
||||
@@ -118,10 +115,10 @@ test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
},
|
||||
})
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [`${relativePath}/child.vhd`, `${relativePath}/abandoned.vhd`],
|
||||
vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
|
||||
}),
|
||||
{ flags: 'w' }
|
||||
)
|
||||
@@ -132,13 +129,13 @@ test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
|
||||
|
||||
const detectedMissingParent = logged.match(/parent disk is missing/g) || []
|
||||
assert.equal(detectedMissingParent.length, 1) // only one vhd should have been deleted
|
||||
const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
|
||||
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
|
||||
})
|
||||
|
||||
test('it remove backup metadata referencing a missing vhd in delta backup', async () => {
|
||||
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
|
||||
// create a metadata file marking child and orphan as ok
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
@@ -170,7 +167,7 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
|
||||
logged += message + '\n'
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
|
||||
let matched = logged.match(/deleting unused disk/g) || []
|
||||
let matched = logged.match(/deleting unused VHD/g) || []
|
||||
assert.equal(matched.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// a missing vhd cause clean to remove all vhds
|
||||
@@ -179,9 +176,9 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${relativePath}/deleted.vhd`, // in metadata but missing from disk
|
||||
`${relativePath}/orphan.vhd`,
|
||||
`${relativePath}/child.vhd`,
|
||||
`deleted.vhd`, // in metadata but not in vhds
|
||||
`orphan.vhd`,
|
||||
`child.vhd`,
|
||||
// abandoned.vhd is not here anymore
|
||||
],
|
||||
}),
|
||||
@@ -189,7 +186,7 @@ test('it remove backup metadata referencing a missing vhd in delta backup', asyn
|
||||
)
|
||||
logged = ''
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
|
||||
matched = logged.match(/deleting unused disk/g) || []
|
||||
matched = logged.match(/deleting unused VHD/g) || []
|
||||
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
|
||||
})
|
||||
|
||||
@@ -236,7 +233,7 @@ test('it merges delta of non destroyed chain', async () => {
|
||||
const result = await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
|
||||
assert.ok(result.size > 0, `no merged size`)
|
||||
const [merging] = logged
|
||||
assert.equal(merging, `Disk chain needs merging`)
|
||||
assert.equal(merging, `merging VHD chain`)
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children after the merge
|
||||
@@ -454,8 +451,8 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
await adapter.cleanVm(rootPath, { remove: true, merge: true, logWarn: () => {}, lock: false })
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size is the sum of the merged chains' final on-disk sizes:
|
||||
assert.deepEqual(metadata.size, 6501888)
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
assert.deepEqual(metadata.size, 4404224)
|
||||
|
||||
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
@@ -484,12 +481,32 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
})
|
||||
test('it cleans orphan merge states ', async () => {
|
||||
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({ mode: 'delta', vhds: [`${relativePath}/nonexistent.vhd`] })
|
||||
)
|
||||
|
||||
await adapter.cleanVm(rootPath, { remove: true, logWarn: () => {}, lock: false })
|
||||
|
||||
assert.deepEqual(await handler.list(basePath), [])
|
||||
})
|
||||
|
||||
test('check Aliases should work alone', async () => {
|
||||
await handler.mkdir('vhds')
|
||||
await handler.mkdir('vhds/data')
|
||||
await generateVhd(`vhds/data/ok.vhd`)
|
||||
await VhdAbstract.createAlias(handler, 'vhds/ok.alias.vhd', 'vhds/data/ok.vhd')
|
||||
|
||||
await VhdAbstract.createAlias(handler, 'vhds/missingData.alias.vhd', 'vhds/data/nonexistent.vhd')
|
||||
|
||||
await generateVhd(`vhds/data/missingalias.vhd`)
|
||||
|
||||
await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', {
|
||||
remove: true,
|
||||
handler,
|
||||
logWarn: () => {},
|
||||
})
|
||||
|
||||
// only ok have survived
|
||||
const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
|
||||
assert.equal(alias.length, 1)
|
||||
|
||||
const data = await handler.list('vhds/data')
|
||||
assert.equal(data.length, 1)
|
||||
})
|
||||
|
||||
628
@xen-orchestra/backups/_cleanVm.mjs
Normal file
628
@xen-orchestra/backups/_cleanVm.mjs
Normal file
@@ -0,0 +1,628 @@
|
||||
import * as UUID from 'uuid'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { Constants, openVhd, VhdAbstract } from 'vhd-lib'
|
||||
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
import { basename, dirname, resolve } from 'node:path'
|
||||
import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
|
||||
import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
|
||||
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
import { Task } from '@vates/task'
|
||||
import handlerPath from '@xen-orchestra/fs/path'
|
||||
|
||||
const { DISK_TYPES } = Constants
|
||||
|
||||
// chain is [ ancestor, child_1, ..., child_n ]
|
||||
async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
|
||||
logInfo(`merging VHD chain`, { chain })
|
||||
|
||||
let done, total
|
||||
const handle = setInterval(() => {
|
||||
if (done !== undefined) {
|
||||
logInfo('merge in progress', {
|
||||
done,
|
||||
parent: chain[0],
|
||||
progress: Math.round((100 * done) / total),
|
||||
total,
|
||||
})
|
||||
}
|
||||
}, 10e3)
|
||||
try {
|
||||
const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
|
||||
|
||||
const childDisks = []
|
||||
for (const path of chain) {
|
||||
childDisks.push(new RemoteVhdDisk({ handler, path }))
|
||||
}
|
||||
const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
|
||||
logInfo,
|
||||
mergeBlockConcurrency,
|
||||
onProgress({ done: d, total: t }) {
|
||||
done = d
|
||||
total = t
|
||||
},
|
||||
removeUnused: remove,
|
||||
})
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
|
||||
await parentDisk.init({ force: isResumingMerge })
|
||||
await childDiskChain.init({ force: isResumingMerge })
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
|
||||
|
||||
return result
|
||||
} finally {
|
||||
clearInterval(handle)
|
||||
}
|
||||
}
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
|
||||
|
||||
const listVhds = async (handler, vmDir, logWarn) => {
|
||||
const vhds = new Set()
|
||||
const aliases = {}
|
||||
const interruptedVhds = new Map()
|
||||
const checksums = new Set()
|
||||
|
||||
await asyncMap(
|
||||
await handler.list(`${vmDir}/vdis`, {
|
||||
ignoreMissing: true,
|
||||
prependDir: true,
|
||||
}),
|
||||
async jobDir =>
|
||||
asyncMap(
|
||||
await handler.list(jobDir, {
|
||||
prependDir: true,
|
||||
}),
|
||||
async vdiDir => {
|
||||
const list = await handler.list(vdiDir, {
|
||||
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
|
||||
})
|
||||
aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
|
||||
|
||||
await asyncMap(list, async file => {
|
||||
if (isVhdSumFile(file)) {
|
||||
checksums.add(`${vdiDir}/${file}`)
|
||||
return
|
||||
}
|
||||
const res = INTERRUPTED_VHDS_REG.exec(file)
|
||||
if (res === null) {
|
||||
vhds.add(`${vdiDir}/${file}`)
|
||||
} else {
|
||||
try {
|
||||
const mergeState = JSON.parse(await handler.readFile(`${vdiDir}/${file}`))
|
||||
interruptedVhds.set(`${vdiDir}/${res[1]}`, {
|
||||
statePath: `${vdiDir}/${file}`,
|
||||
chain: mergeState.chain,
|
||||
})
|
||||
} catch (error) {
|
||||
// fall back to a non resuming merge
|
||||
vhds.add(`${vdiDir}/${file}`)
|
||||
logWarn('failed to read existing merge state', { path: file, error })
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return { vhds, interruptedVhds, aliases, checksums }
|
||||
}
|
||||
|
||||
export async function checkAliases(
|
||||
aliasPaths,
|
||||
targetDataRepository,
|
||||
{ handler, logInfo = noop, logWarn = console.warn, remove = false }
|
||||
) {
|
||||
const aliasFound = []
|
||||
for (const alias of aliasPaths) {
|
||||
let target
|
||||
try {
|
||||
target = await resolveVhdAlias(handler, alias)
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
logWarn('missing target of alias', { alias })
|
||||
if (remove) {
|
||||
logInfo('removing alias and non VHD target', { alias, target })
|
||||
await handler.unlink(alias)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (err.code === 'EISDIR') {
|
||||
logWarn('alias is a vhd directory', { alias })
|
||||
if (remove) {
|
||||
logInfo('removing vhd directory named as alias', { alias, target })
|
||||
await VhdAbstract.unlink(handler, alias)
|
||||
}
|
||||
continue
|
||||
}
|
||||
logWarn('unhandled error while checking alias', { alias, err })
|
||||
continue
|
||||
}
|
||||
|
||||
if (target === '') {
|
||||
logWarn('empty target for alias ', { alias })
|
||||
if (remove) {
|
||||
logInfo('removing alias and non VHD target', { alias, target })
|
||||
await handler.unlink(alias)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (!isVhdFile(target)) {
|
||||
logWarn('alias references non VHD target', { alias, target })
|
||||
if (remove) {
|
||||
logInfo('removing alias and non VHD target', { alias, target })
|
||||
await handler.unlink(target)
|
||||
await handler.unlink(alias)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const { dispose } = await openVhd(handler, target)
|
||||
try {
|
||||
await dispose()
|
||||
} catch (e) {
|
||||
// error during dispose should not trigger a deletion
|
||||
}
|
||||
} catch (error) {
|
||||
logWarn('missing or broken alias target', { alias, target, error })
|
||||
if (remove) {
|
||||
try {
|
||||
await VhdAbstract.unlink(handler, alias)
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logWarn('error deleting alias target', { alias, target, error })
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
aliasFound.push(resolve('/', target))
|
||||
}
|
||||
|
||||
const vhds = await handler.list(targetDataRepository, {
|
||||
ignoreMissing: true,
|
||||
prependDir: true,
|
||||
})
|
||||
|
||||
await asyncMap(vhds, async path => {
|
||||
if (!aliasFound.includes(path)) {
|
||||
logWarn('no alias references VHD', { path })
|
||||
if (remove) {
|
||||
logInfo('deleting unused VHD', { path })
|
||||
await VhdAbstract.unlink(handler, path)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const defaultMergeLimiter = limitConcurrency(1)
|
||||
|
||||
export async function cleanVm(
|
||||
vmDir,
|
||||
{
|
||||
fixMetadata,
|
||||
remove = false,
|
||||
removeTmp = remove,
|
||||
merge,
|
||||
mergeBlockConcurrency,
|
||||
mergeLimiter = defaultMergeLimiter,
|
||||
logInfo = noop,
|
||||
logWarn = console.warn,
|
||||
}
|
||||
) {
|
||||
const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
|
||||
|
||||
const handler = this._handler
|
||||
|
||||
const vhdsToJSons = new Set()
|
||||
const vhdById = new Map()
|
||||
const vhdParents = { __proto__: null }
|
||||
const vhdChildren = { __proto__: null }
|
||||
|
||||
const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
|
||||
|
||||
// from 5.110 to 5.113 we computed checksum for vhd file
|
||||
// but never used nor removed them
|
||||
await asyncMap(checksums, async path => {
|
||||
if (remove) {
|
||||
logInfo('deleting checksum file ', { path })
|
||||
return handler.unlink(path)
|
||||
}
|
||||
})
|
||||
|
||||
// remove broken VHDs
|
||||
await asyncMap(vhds, async path => {
|
||||
if (removeTmp && basename(path)[0] === '.') {
|
||||
logInfo('deleting temporary VHD', { path })
|
||||
return VhdAbstract.unlink(handler, path)
|
||||
}
|
||||
try {
|
||||
await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
|
||||
if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
|
||||
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
|
||||
vhdParents[path] = parent
|
||||
if (parent in vhdChildren) {
|
||||
const error = new Error('this script does not support multiple VHD children')
|
||||
error.parent = parent
|
||||
error.child1 = vhdChildren[parent]
|
||||
error.child2 = path
|
||||
throw error // should we throw?
|
||||
}
|
||||
vhdChildren[parent] = path
|
||||
}
|
||||
// Detect VHDs with the same UUIDs
|
||||
//
|
||||
// Due to a bug introduced in a1bcd35e2
|
||||
const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
|
||||
let vhdKept = vhd
|
||||
if (duplicate !== undefined) {
|
||||
logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
|
||||
if (duplicate.containsAllDataOf(vhd)) {
|
||||
logWarn(`should delete ${path}`)
|
||||
vhdKept = duplicate
|
||||
vhds.delete(path)
|
||||
} else if (vhd.containsAllDataOf(duplicate)) {
|
||||
logWarn(`should delete ${duplicate._path}`)
|
||||
vhds.delete(duplicate._path)
|
||||
} else {
|
||||
logWarn('same ids but different content')
|
||||
}
|
||||
}
|
||||
vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
|
||||
})
|
||||
} catch (error) {
|
||||
vhds.delete(path)
|
||||
logWarn('VHD check error', { path, error })
|
||||
if (error?.code === 'ERR_ASSERTION' && remove) {
|
||||
logInfo('deleting broken VHD', { path })
|
||||
return VhdAbstract.unlink(handler, path)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// remove interrupted merge states for missing VHDs
|
||||
for (const interruptedVhd of interruptedVhds.keys()) {
|
||||
if (!vhds.has(interruptedVhd)) {
|
||||
const { statePath } = interruptedVhds.get(interruptedVhd)
|
||||
interruptedVhds.delete(interruptedVhd)
|
||||
|
||||
logWarn('orphan merge state', {
|
||||
mergeStatePath: statePath,
|
||||
missingVhdPath: interruptedVhd,
|
||||
})
|
||||
if (remove) {
|
||||
logInfo('deleting orphan merge state', { statePath })
|
||||
await handler.unlink(statePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if alias are correct
|
||||
// check if all vhd in data subfolder have a corresponding alias
|
||||
await asyncMap(Object.keys(aliases), async dir => {
|
||||
await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
|
||||
})
|
||||
|
||||
// remove VHDs with missing ancestors
|
||||
{
|
||||
const deletions = []
|
||||
|
||||
// return true if the VHD has been deleted or is missing
|
||||
const deleteIfOrphan = vhdPath => {
|
||||
const parent = vhdParents[vhdPath]
|
||||
if (parent === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
// no longer needs to be checked
|
||||
delete vhdParents[vhdPath]
|
||||
|
||||
deleteIfOrphan(parent)
|
||||
|
||||
if (!vhds.has(parent)) {
|
||||
vhds.delete(vhdPath)
|
||||
|
||||
logWarn('parent VHD is missing', { parent, child: vhdPath })
|
||||
if (remove) {
|
||||
logInfo('deleting orphan VHD', { path: vhdPath })
|
||||
deletions.push(VhdAbstract.unlink(handler, vhdPath))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// > A property that is deleted before it has been visited will not be
|
||||
// > visited later.
|
||||
// >
|
||||
// > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
|
||||
for (const child in vhdParents) {
|
||||
deleteIfOrphan(child)
|
||||
}
|
||||
|
||||
await Promise.all(deletions)
|
||||
}
|
||||
|
||||
const jsons = new Set()
|
||||
const xvas = new Set()
|
||||
const xvaSums = []
|
||||
const entries = await handler.list(vmDir, {
|
||||
prependDir: true,
|
||||
})
|
||||
entries.forEach(path => {
|
||||
if (isMetadataFile(path)) {
|
||||
jsons.add(path)
|
||||
} else if (isXvaFile(path)) {
|
||||
xvas.add(path)
|
||||
} else if (isXvaSumFile(path)) {
|
||||
xvaSums.push(path)
|
||||
}
|
||||
})
|
||||
|
||||
const cachePath = vmDir + '/cache.json.gz'
|
||||
|
||||
let mustRegenerateCache
|
||||
{
|
||||
const cache = await this._readCache(cachePath)
|
||||
const actual = cache === undefined ? 0 : Object.keys(cache).length
|
||||
const expected = jsons.size
|
||||
|
||||
mustRegenerateCache = actual !== expected
|
||||
if (mustRegenerateCache) {
|
||||
logWarn('unexpected number of entries in backup cache', { path: cachePath, actual, expected })
|
||||
}
|
||||
}
|
||||
|
||||
await asyncMap(xvas, async path => {
|
||||
// check is not good enough to delete the file, the best we can do is report
|
||||
// it
|
||||
if (!(await this.isValidXva(path))) {
|
||||
logWarn('XVA might be broken', { path })
|
||||
}
|
||||
})
|
||||
|
||||
const unusedVhds = new Set(vhds)
|
||||
const unusedXvas = new Set(xvas)
|
||||
|
||||
const backups = new Map()
|
||||
|
||||
// compile the list of unused XVAs and VHDs, and remove backup metadata which
|
||||
// reference a missing XVA/VHD
|
||||
await asyncMap(jsons, async json => {
|
||||
let metadata
|
||||
try {
|
||||
metadata = JSON.parse(await handler.readFile(json))
|
||||
} catch (error) {
|
||||
logWarn('failed to read backup metadata', { path: json, error })
|
||||
jsons.delete(json)
|
||||
return
|
||||
}
|
||||
|
||||
let isBackupComplete
|
||||
|
||||
const { mode } = metadata
|
||||
if (mode === 'full') {
|
||||
const linkedXva = resolve('/', vmDir, metadata.xva)
|
||||
isBackupComplete = xvas.has(linkedXva)
|
||||
if (isBackupComplete) {
|
||||
unusedXvas.delete(linkedXva)
|
||||
} else {
|
||||
logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
|
||||
}
|
||||
} else if (mode === 'delta') {
|
||||
const linkedVhds = (() => {
|
||||
const { vhds } = metadata
|
||||
return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
|
||||
})()
|
||||
|
||||
const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
|
||||
isBackupComplete = missingVhds.length === 0
|
||||
|
||||
// FIXME: find better approach by keeping as much of the backup as
|
||||
// possible (existing disks) even if one disk is missing
|
||||
if (isBackupComplete) {
|
||||
linkedVhds.forEach(_ => unusedVhds.delete(_))
|
||||
linkedVhds.forEach(path => {
|
||||
vhdsToJSons[path] = json
|
||||
})
|
||||
} else {
|
||||
logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
|
||||
}
|
||||
}
|
||||
|
||||
if (isBackupComplete) {
|
||||
backups.set(json, metadata)
|
||||
} else {
|
||||
jsons.delete(json)
|
||||
if (remove) {
|
||||
logInfo('deleting incomplete backup', { backup: json })
|
||||
mustRegenerateCache = true
|
||||
await handler.unlink(json)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// TODO: parallelize by vm/job/vdi
|
||||
const unusedVhdsDeletion = []
|
||||
const toMerge = []
|
||||
{
|
||||
// VHD chains (as list from oldest to most recent) to merge indexed by most recent
|
||||
// ancestor
|
||||
const vhdChainsToMerge = { __proto__: null }
|
||||
|
||||
const toCheck = new Set(unusedVhds)
|
||||
|
||||
const getUsedChildChainOrDelete = vhd => {
|
||||
if (vhd in vhdChainsToMerge) {
|
||||
const chain = vhdChainsToMerge[vhd]
|
||||
delete vhdChainsToMerge[vhd]
|
||||
return chain
|
||||
}
|
||||
|
||||
if (!unusedVhds.has(vhd)) {
|
||||
return [vhd]
|
||||
}
|
||||
|
||||
// no longer needs to be checked
|
||||
toCheck.delete(vhd)
|
||||
|
||||
const child = vhdChildren[vhd]
|
||||
if (child !== undefined) {
|
||||
const chain = getUsedChildChainOrDelete(child)
|
||||
if (chain !== undefined) {
|
||||
chain.unshift(vhd)
|
||||
return chain
|
||||
}
|
||||
}
|
||||
|
||||
// no warning because a VHD can be unused for perfectly good reasons,
|
||||
// e.g. the corresponding backup (metadata file) has been deleted
|
||||
if (remove) {
|
||||
logInfo('deleting unused VHD', { path: vhd })
|
||||
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
|
||||
}
|
||||
}
|
||||
|
||||
toCheck.forEach(vhd => {
|
||||
vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
|
||||
})
|
||||
|
||||
// merge interrupted VHDs
|
||||
for (const parent of interruptedVhds.keys()) {
|
||||
// before #6349 the chain wasn't in the mergeState
|
||||
const { chain, statePath } = interruptedVhds.get(parent)
|
||||
if (chain === undefined) {
|
||||
vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
|
||||
} else {
|
||||
vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
|
||||
}
|
||||
}
|
||||
|
||||
Object.values(vhdChainsToMerge).forEach(chain => {
|
||||
if (chain !== undefined) {
|
||||
toMerge.push(chain)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
let totalMergedDataSize = 0
|
||||
const metadataWithMergedVhd = {}
|
||||
const doMerge = async () => {
|
||||
await asyncMap(toMerge, async chain => {
|
||||
const { finalDiskSize, mergedDataSize } = await limitedMergeVhdChain(handler, chain, {
|
||||
logInfo,
|
||||
logWarn,
|
||||
remove,
|
||||
mergeBlockConcurrency,
|
||||
})
|
||||
totalMergedDataSize += mergedDataSize
|
||||
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
|
||||
})
|
||||
|
||||
return { size: totalMergedDataSize }
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
...unusedVhdsDeletion,
|
||||
toMerge.length !== 0 && (merge ? Task.run({ properties: { name: 'merge' } }, doMerge) : () => Promise.resolve()),
|
||||
asyncMap(unusedXvas, path => {
|
||||
logWarn('unused XVA', { path })
|
||||
if (remove) {
|
||||
logInfo('deleting unused XVA', { path })
|
||||
return handler.unlink(path)
|
||||
}
|
||||
}),
|
||||
asyncMap(xvaSums, path => {
|
||||
// no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
|
||||
if (!xvas.has(path.slice(0, -'.checksum'.length))) {
|
||||
logInfo('unused XVA checksum', { path })
|
||||
if (remove) {
|
||||
logInfo('deleting unused XVA checksum', { path })
|
||||
return handler.unlink(path)
|
||||
}
|
||||
}
|
||||
}),
|
||||
])
|
||||
|
||||
// update size for delta metadata with merged VHD
|
||||
// check for the other that the size is the same as the real file size
|
||||
await asyncMap(jsons, async metadataPath => {
|
||||
const metadata = backups.get(metadataPath)
|
||||
|
||||
let fileSystemSize
|
||||
const mergedSize = metadataWithMergedVhd[metadataPath]
|
||||
|
||||
const { mode, size, xva } = metadata
|
||||
|
||||
try {
|
||||
if (mode === 'full') {
|
||||
// a full backup : check size
|
||||
const linkedXva = resolve('/', vmDir, xva)
|
||||
try {
|
||||
fileSystemSize = await handler.getSize(linkedXva)
|
||||
if (fileSystemSize !== size && fileSystemSize !== undefined) {
|
||||
logWarn('cleanVm: incorrect backup size in metadata', {
|
||||
path: metadataPath,
|
||||
actual: size ?? 'none',
|
||||
expected: fileSystemSize,
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
// will fail with encrypted remote
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logWarn('failed to get backup size', { backup: metadataPath, error })
|
||||
return
|
||||
}
|
||||
|
||||
// systematically update size and differentials after a merge
|
||||
|
||||
// @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
|
||||
if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
|
||||
metadata.size = mergedSize ?? fileSystemSize ?? size
|
||||
|
||||
if (mergedSize) {
|
||||
// all disks are now key disk
|
||||
metadata.isVhdDifferencing = {}
|
||||
for (const id of Object.keys(metadata.vdis ?? {})) {
|
||||
metadata.isVhdDifferencing[id] = false
|
||||
}
|
||||
}
|
||||
mustRegenerateCache = true
|
||||
try {
|
||||
await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
|
||||
} catch (error) {
|
||||
logWarn('failed to update backup size in metadata', { path: metadataPath, error })
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (mustRegenerateCache) {
|
||||
const cache = {}
|
||||
for (const [path, content] of backups.entries()) {
|
||||
cache[path] = {
|
||||
_filename: path,
|
||||
id: path,
|
||||
...content,
|
||||
}
|
||||
}
|
||||
await this._writeCache(cachePath, cache)
|
||||
}
|
||||
|
||||
return {
|
||||
// boolean whether some VHDs were merged (or should be merged)
|
||||
merge: toMerge.length !== 0,
|
||||
size: totalMergedDataSize,
|
||||
}
|
||||
}
|
||||
@@ -85,7 +85,6 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
|
||||
export function resetVmOtherConfig(xapi, vmRef) {
|
||||
return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
|
||||
return xapi.setFieldEntries(type, ref, 'other_config', {
|
||||
[CONTENT_KEY]: null,
|
||||
[COPY_OF]: null,
|
||||
[DATETIME]: null,
|
||||
[DELTA_CHAIN_LENGTH]: null,
|
||||
@@ -93,7 +92,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
|
||||
[JOB_ID]: null,
|
||||
[SCHEDULE_ID]: null,
|
||||
[VM_UUID]: null,
|
||||
|
||||
// REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
|
||||
})
|
||||
})
|
||||
|
||||
@@ -71,7 +71,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
const queue = new Set(vmsUuids)
|
||||
const taskByVmId = {}
|
||||
const nTriesByVmId = {}
|
||||
const vmErrors = []
|
||||
|
||||
const handleVm = vmUuid => {
|
||||
if (nTriesByVmId[vmUuid] === undefined) {
|
||||
@@ -127,7 +126,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
return task.success(result)
|
||||
}
|
||||
if (isLastRun) {
|
||||
vmErrors.push(taskError)
|
||||
return task.failure(taskError)
|
||||
}
|
||||
// don't end the task
|
||||
@@ -150,8 +148,6 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
|
||||
await asyncMapSettled(vmIds, _handleVm)
|
||||
}
|
||||
|
||||
this._throwVmErrors(vmErrors)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -98,7 +98,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
const queue = new Set(vmIds)
|
||||
const taskByVmId = {}
|
||||
const nTriesByVmId = {}
|
||||
const vmErrors = []
|
||||
|
||||
const handleVm = vmUuid => {
|
||||
const getVmTask = () => {
|
||||
@@ -176,19 +175,11 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
} else {
|
||||
// ending the task with error or not ending the task
|
||||
vmBackupFailed(taskError, task)
|
||||
if (isLastRun) {
|
||||
vmErrors.push(taskError)
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch(noop) // errors are handled by logs
|
||||
}),
|
||||
error => {
|
||||
vmBackupFailed(error, getVmTask())
|
||||
if (isLastRun) {
|
||||
vmErrors.push(error)
|
||||
}
|
||||
}
|
||||
error => vmBackupFailed(error, getVmTask())
|
||||
)
|
||||
}
|
||||
const { concurrency } = settings
|
||||
@@ -200,8 +191,6 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
|
||||
await asyncMapSettled(vmIds, _handleVm)
|
||||
}
|
||||
|
||||
this._throwVmErrors(vmErrors)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -11,22 +11,7 @@ export const DEFAULT_SETTINGS = {
|
||||
reportWhen: 'failure',
|
||||
}
|
||||
|
||||
export class AggregateError extends Error {
|
||||
constructor(errors, message) {
|
||||
super(message)
|
||||
this.errors = errors
|
||||
}
|
||||
}
|
||||
|
||||
export const Abstract = class AbstractRunner {
|
||||
_throwVmErrors(errors) {
|
||||
if (errors.length === 1) {
|
||||
throw errors[0]
|
||||
} else if (errors.length > 1) {
|
||||
throw new AggregateError(errors, `${errors.length} VMs failed`)
|
||||
}
|
||||
}
|
||||
|
||||
constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
|
||||
this._config = config
|
||||
this._getRecord = getConnectedRecord
|
||||
|
||||
@@ -8,7 +8,6 @@ import { exportIncrementalVm } from '../../_incrementalVm.mjs'
|
||||
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
|
||||
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
|
||||
import {
|
||||
CONTENT_KEY,
|
||||
DATETIME,
|
||||
DELTA_CHAIN_LENGTH,
|
||||
EXPORTED_SUCCESSFULLY,
|
||||
@@ -102,136 +101,63 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups snapshot VDIs by their DATETIME other_config value and sorts them most-recent-first.
|
||||
*
|
||||
* @param {import('@vates/types').XenApiVdi[]} snapshotVdis
|
||||
* @returns {{ datetime: string, vdis: import('@vates/types').XenApiVdi[] }[]}
|
||||
*/
|
||||
_groupSnapshotVdisByDatetime(snapshotVdis) {
|
||||
const byDatetime = new Map()
|
||||
for (const vdi of snapshotVdis) {
|
||||
const datetime = vdi.other_config[DATETIME]
|
||||
if (!byDatetime.has(datetime)) {
|
||||
byDatetime.set(datetime, [])
|
||||
}
|
||||
byDatetime.get(datetime).push(vdi)
|
||||
}
|
||||
// sort them by decreasing datetime
|
||||
return [...byDatetime.entries()]
|
||||
.sort(([a], [b]) => b.localeCompare(a))
|
||||
.map(([datetime, vdis]) => ({ datetime, vdis }))
|
||||
}
|
||||
async _selectBaseVm() {
|
||||
const xapi = this._xapi
|
||||
|
||||
// filter _jobSnapshotVdis to have only the last successfully exported vdi
|
||||
// compute delta chain length
|
||||
// fill baseUuidToSrcVdi with this data
|
||||
|
||||
const exportedVdis = this._jobSnapshotVdis.filter(
|
||||
_ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
|
||||
)
|
||||
let lastSuccessfullBackup
|
||||
let lastExportedVdis = []
|
||||
for (const exportedVdi of exportedVdis) {
|
||||
if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
|
||||
lastExportedVdis = []
|
||||
lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
|
||||
}
|
||||
if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
|
||||
lastExportedVdis.push(exportedVdi)
|
||||
}
|
||||
}
|
||||
|
||||
this._baseVdis = {}
|
||||
if (lastExportedVdis.length === 0) {
|
||||
debug('no base VDIS found', {
|
||||
jobLength: this._jobSnapshotVdis.length,
|
||||
lastSuccessfullBackup,
|
||||
exportedLength: exportedVdis.length,
|
||||
})
|
||||
return
|
||||
}
|
||||
const deltaChainLength = Math.max(
|
||||
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
|
||||
)
|
||||
|
||||
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
|
||||
|
||||
/**
|
||||
* For each snapshot VDI in `snapshotVdis`, maps it to its live source VDI,
|
||||
* attaches its CONTENT_KEY when present, calls checkBaseVdis on all writers,
|
||||
* and returns both the full mapping and the subset confirmed present by writers.
|
||||
*
|
||||
* @param {import('@vates/types').XenApiVdi[]} snapshotVdis - Snapshot VDI records to use as base candidates
|
||||
* @param {Record<string, object>} srcVdisByRef - Live VM VDI records keyed by $ref
|
||||
* @param {{ useContentKey?: boolean }} [options]
|
||||
* @returns {Promise<{ presentBaseVdis: Map<string, string>, baseUuidToSrcVdiUuid: Map<string, string>}>}
|
||||
*/
|
||||
async _checkBaseVdis(snapshotVdis, srcVdisByRef, { useContentKey = false } = {}) {
|
||||
const baseUuidToSrcVdiUuid = new Map()
|
||||
const baseUuidToContentKey = new Map()
|
||||
for (const snapshotVdi of snapshotVdis) {
|
||||
const baseUuid = snapshotVdi.uuid
|
||||
const srcVdi = srcVdisByRef[snapshotVdi.snapshot_of]
|
||||
for (const lastExportedVdi of lastExportedVdis) {
|
||||
const baseUuid = lastExportedVdi.uuid
|
||||
const snapshotOf = lastExportedVdi.snapshot_of
|
||||
const srcVdi = srcVdis[snapshotOf]
|
||||
if (srcVdi !== undefined) {
|
||||
baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
|
||||
if (useContentKey) {
|
||||
const contentKey = snapshotVdi.other_config[CONTENT_KEY]
|
||||
if (contentKey !== undefined) {
|
||||
baseUuidToContentKey.set(baseUuid, contentKey)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
debug('ignore snapshot VDI because no longer present on VM', { vdi: baseUuid })
|
||||
debug('ignore snapshot VDI because no longer present on VM', {
|
||||
vdi: baseUuid,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
|
||||
await this._callWriters(
|
||||
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseUuidToContentKey),
|
||||
writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
|
||||
'writer.checkBaseVdis()',
|
||||
false
|
||||
)
|
||||
return { presentBaseVdis, baseUuidToSrcVdiUuid }
|
||||
}
|
||||
|
||||
async _selectBaseVm() {
|
||||
const xapi = this._xapi
|
||||
|
||||
const exportedVdis = this._jobSnapshotVdis.filter(
|
||||
_ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
|
||||
)
|
||||
const grouped = this._groupSnapshotVdisByDatetime(exportedVdis)
|
||||
|
||||
this._baseVdis = {}
|
||||
if (grouped.length === 0) {
|
||||
debug('no base VDIs found', {
|
||||
jobLength: this._jobSnapshotVdis.length,
|
||||
exportedLength: exportedVdis.length,
|
||||
})
|
||||
}
|
||||
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
|
||||
let lastExportedVdis = []
|
||||
let presentBaseVdis = new Map()
|
||||
let baseUuidToSrcVdiUuid = new Map()
|
||||
let deltaChainLength
|
||||
|
||||
if (grouped.length > 0) {
|
||||
lastExportedVdis = grouped[0].vdis
|
||||
// only compute it on the _jobSnapshotVdis , not on the reused snapshot from CONTENT_KEYS
|
||||
deltaChainLength = Math.max(
|
||||
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
|
||||
)
|
||||
;({ presentBaseVdis, baseUuidToSrcVdiUuid } = await this._checkBaseVdis(lastExportedVdis, srcVdis))
|
||||
}
|
||||
|
||||
// if we don't find any candidates
|
||||
// if there is only one writer of type IncrementalXapiWriter
|
||||
// allow the reuse of any matching snapshot of this VM even from another job
|
||||
if (
|
||||
presentBaseVdis.size === 0 &&
|
||||
this._writers.size === 1 &&
|
||||
[...this._writers][0] instanceof IncrementalXapiWriter
|
||||
) {
|
||||
debug('fallback to content key')
|
||||
const candidates = {}
|
||||
for (const srcVdi of Object.values(srcVdis)) {
|
||||
const snapshots = srcVdi.$snapshots
|
||||
debug(`${srcVdi.uuid} got ${snapshots.length} snapshots`)
|
||||
for (const snapshot of snapshots) {
|
||||
if (DATETIME in snapshot.other_config) {
|
||||
candidates[snapshot.uuid] = snapshot
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug('got candidates ', Object.keys(candidates).length)
|
||||
this._filterValidSnapshotVdis(candidates)
|
||||
debug('got filtered ', Object.keys(candidates).length)
|
||||
for (const group of this._groupSnapshotVdisByDatetime(Object.values(candidates))) {
|
||||
debug('check by datetime ', [...group.vdis].length)
|
||||
debug(group.vdis)
|
||||
const result = await this._checkBaseVdis(group.vdis, srcVdis, { useContentKey: true })
|
||||
debug('result ', result)
|
||||
if (result.presentBaseVdis.size > 0) {
|
||||
lastExportedVdis = group.vdis
|
||||
|
||||
presentBaseVdis = result.presentBaseVdis
|
||||
baseUuidToSrcVdiUuid = result.baseUuidToSrcVdiUuid
|
||||
|
||||
break
|
||||
}
|
||||
debug('try next group if any')
|
||||
}
|
||||
} else {
|
||||
debug(' no fallback ')
|
||||
}
|
||||
|
||||
if (presentBaseVdis.size === 0) {
|
||||
debug('no base VM found')
|
||||
|
||||
@@ -236,7 +236,28 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// ensure they are attached to only one vm snapshot
|
||||
// ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
|
||||
|
||||
_filterValidSnapshotVdis(vdiCandidates) {
|
||||
async _fetchJobSnapshots() {
|
||||
const jobId = this._jobId
|
||||
const xapi = this._xapi
|
||||
|
||||
const vdiCandidates = {}
|
||||
const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
|
||||
Object.values(xapi.objects.indexes.type.VDI)
|
||||
.filter(_ => !!_) // filter nullish
|
||||
.filter(({ other_config, snapshot_of }) => {
|
||||
return (
|
||||
vdiUuids.includes(snapshot_of) &&
|
||||
other_config[JOB_ID] === jobId &&
|
||||
other_config[VM_UUID] === this._vm.uuid &&
|
||||
other_config[COPY_OF] === undefined
|
||||
)
|
||||
})
|
||||
.forEach(vdi => {
|
||||
vdiCandidates[vdi.uuid] = vdi
|
||||
})
|
||||
|
||||
// check that user snapshots are clean
|
||||
|
||||
for (const vdi of Object.values(vdiCandidates)) {
|
||||
// cbt metadata are always considered linked to a backup job
|
||||
// if they have the right other_config
|
||||
@@ -258,7 +279,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
const userVms = vbds.map(({ $VM }) => $VM)
|
||||
if (vbds.length > 1) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, userVms }
|
||||
)
|
||||
@@ -273,7 +294,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// => we exclude these from the backup processing
|
||||
if (vm.$snapshot_of === undefined) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
@@ -290,7 +311,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
|
||||
) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
@@ -307,7 +328,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
.forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
|
||||
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
|
||||
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
|
||||
is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm, vbds, outOfSnapshotsVdi }
|
||||
)
|
||||
@@ -315,47 +336,8 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async _fetchJobSnapshots() {
|
||||
const jobId = this._jobId
|
||||
const xapi = this._xapi
|
||||
|
||||
const vdiCandidates = {}
|
||||
const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
|
||||
Object.values(xapi.objects.indexes.type.VDI)
|
||||
.filter(_ => !!_) // filter nullish
|
||||
.filter(({ other_config, snapshot_of }) => {
|
||||
return (
|
||||
vdiUuids.includes(snapshot_of) &&
|
||||
other_config[JOB_ID] === jobId &&
|
||||
other_config[VM_UUID] === this._vm.uuid &&
|
||||
other_config[COPY_OF] === undefined
|
||||
)
|
||||
})
|
||||
.forEach(vdi => {
|
||||
vdiCandidates[vdi.uuid] = vdi
|
||||
})
|
||||
|
||||
this._filterValidSnapshotVdis(vdiCandidates)
|
||||
|
||||
this._jobSnapshotVdis = Object.values(vdiCandidates)
|
||||
|
||||
// For VMs with no disks, retention must be tracked directly on VM snapshots
|
||||
// since there are no VDIs to anchor the discovery.
|
||||
if (vdiUuids.length === 0) {
|
||||
this._disklessJobSnapshotVms = this._vm.$snapshots
|
||||
.filter(Boolean)
|
||||
.filter(
|
||||
({ other_config, $snapshot_of }) =>
|
||||
$snapshot_of !== undefined &&
|
||||
other_config[JOB_ID] === jobId &&
|
||||
other_config[VM_UUID] === this._vm.uuid &&
|
||||
other_config[COPY_OF] === undefined
|
||||
)
|
||||
} else {
|
||||
this._disklessJobSnapshotVms = []
|
||||
}
|
||||
}
|
||||
|
||||
async _removeUnusedSnapshots() {
|
||||
@@ -367,10 +349,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
await xapi.barrier()
|
||||
// ensure cached object are up to date
|
||||
this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
|
||||
const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
|
||||
|
||||
// get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
|
||||
const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
|
||||
// get the datetime of the most recent snapshot
|
||||
const lastSnapshotDateTime = this._jobSnapshotVdis
|
||||
.map(({ other_config }) => other_config[DATETIME])
|
||||
.sort()
|
||||
.pop()
|
||||
@@ -447,28 +428,6 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
})
|
||||
})
|
||||
|
||||
// Retention for VMs with no disks: VM snapshots are not reachable via VDIs
|
||||
if (disklessVmSnapshots.length > 0) {
|
||||
const snapshotsPerSchedule = groupBy(disklessVmSnapshots, _ => _.other_config[SCHEDULE_ID])
|
||||
await asyncEach(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
|
||||
// we only have one snapshot per date time since it's at the VM level
|
||||
const snapshotPerDatetime = Object.fromEntries(snapshots.map(s => [s.other_config[DATETIME], s.$ref]))
|
||||
const datetimes = Object.keys(snapshotPerDatetime).sort()
|
||||
const settings = {
|
||||
...baseSettings,
|
||||
...allSettings[scheduleId],
|
||||
...allSettings[this._vm.uuid],
|
||||
}
|
||||
const retention = settings.snapshotRetention ?? 0
|
||||
await asyncEach(getOldEntries(retention, datetimes), async datetime => {
|
||||
if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
|
||||
return
|
||||
}
|
||||
await xapi.VM_destroy(snapshotPerDatetime[datetime])
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// list and remove the snapshot were the jobs failed between
|
||||
// makesnapshot and update_other_config
|
||||
const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)
|
||||
|
||||
@@ -229,13 +229,14 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
Object.entries(deltaExport.disks),
|
||||
async ([diskRef, disk]) => {
|
||||
const path = `${this._vmBackupDir}/${vhds[diskRef]}`
|
||||
size += await adapter.writeVhd(path, disk, {
|
||||
await adapter.writeVhd(path, disk, {
|
||||
// no checksum for VHDs, because they will be invalidated by
|
||||
// merges and chains
|
||||
checksum: false,
|
||||
validator: tmpPath => checkVhd(handler, tmpPath),
|
||||
writeBlockConcurrency: this._config.writeBlockConcurrency,
|
||||
})
|
||||
size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
|
||||
},
|
||||
{
|
||||
concurrency: settings.diskPerVmConcurrency,
|
||||
|
||||
@@ -19,14 +19,11 @@ import {
|
||||
REPLICATED_TO_SR_UUID,
|
||||
DATETIME,
|
||||
VM_UUID,
|
||||
CONTENT_KEY,
|
||||
resetVmOtherConfig,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { XapiDiskSource } from '@xen-orchestra/xapi'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { VM_POWER_STATE } from '@vates/types'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiWriter')
|
||||
|
||||
@@ -35,20 +32,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
// Built by checkBaseVdis, consumed by #decorateVmMetadata to set baseVdi.
|
||||
#baseVdisBySourceUuid = new Map()
|
||||
|
||||
/**
|
||||
* Finds VDIs on the target SR that can serve as base for the next delta transfer.
|
||||
*
|
||||
* For each entry in `baseUuidToSrcVdi`, searches the target SR for a matching snapshot
|
||||
* using CONTENT_KEY when available, then falls back to COPY_OF matching for older snapshots.
|
||||
* Entries with no matching base found on the target SR are removed from `baseUuidToSrcVdi`.
|
||||
*
|
||||
* Side-effects: populates `#baseVdisBySourceUuid`; may set `_targetVmRef`.
|
||||
*
|
||||
* @param {Map<string, string>} baseUuidToSrcVdi - Source snapshot UUID → source active VDI UUID. Mutated in place.
|
||||
* @param {Map<string, string>} contentKeys - Source snapshot UUID → CONTENT_KEY value (empty for pre-CONTENT_KEY snapshots).
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi, contentKeys) {
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
const sr = this._sr
|
||||
this.#baseVdisBySourceUuid = new Map()
|
||||
|
||||
@@ -61,45 +45,18 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
// look for the same snapshot
|
||||
// ensure there are no data between the snapshot and the active disk
|
||||
|
||||
const snapshotCandidates = new Map()
|
||||
const snapshotCandidates = sr.$VDIs.filter(vdi => {
|
||||
return (
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[JOB_ID] === this._job.id &&
|
||||
vdi.other_config[VM_UUID] === this._vmUuid &&
|
||||
baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
|
||||
)
|
||||
})
|
||||
debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.length)
|
||||
|
||||
for (const [baseUuid, srcVdiuid] of baseUuidToSrcVdi) {
|
||||
let target
|
||||
|
||||
const contentKey = contentKeys.get(baseUuid)
|
||||
|
||||
if (contentKey !== undefined) {
|
||||
debug('got one content key, look for a candidate')
|
||||
target = sr.$VDIs.find(
|
||||
vdi =>
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[CONTENT_KEY] === contentKey && // &&
|
||||
// ensure we don't replicate on ourself or any of the vdi in the source chain
|
||||
vdi.$snapshot_of.uuid !== srcVdiuid
|
||||
)
|
||||
}
|
||||
|
||||
// fall back for older snapshots
|
||||
if (target === undefined) {
|
||||
debug('content key not here or not found , look by jobid ')
|
||||
target = sr.$VDIs.find(
|
||||
vdi =>
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[JOB_ID] === this._job.id &&
|
||||
vdi.other_config[VM_UUID] === this._vmUuid &&
|
||||
vdi?.other_config[COPY_OF] === baseUuid
|
||||
)
|
||||
}
|
||||
if (target !== undefined) {
|
||||
snapshotCandidates.set(baseUuid, target)
|
||||
}
|
||||
}
|
||||
|
||||
debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.size)
|
||||
|
||||
if (snapshotCandidates.size > 0) {
|
||||
if (snapshotCandidates.length > 0) {
|
||||
// reset before searching for candidates
|
||||
this.#baseVdisBySourceUuid = new Map()
|
||||
this._targetVmRef = undefined
|
||||
@@ -107,12 +64,10 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
for (const [sourceUuid, vdi] of baseVdisBySourceUuid) {
|
||||
this.#baseVdisBySourceUuid.set(sourceUuid, vdi)
|
||||
}
|
||||
debug(' this.#baseVdisBySourceUuid ', this.#baseVdisBySourceUuid.size)
|
||||
if (targetVmRef !== undefined) {
|
||||
this._targetVmRef = targetVmRef
|
||||
}
|
||||
} else {
|
||||
debug('legacy fallback ( no content key ) ')
|
||||
// Legacy fallback (upgrade from pre-6.3): no target snapshots exist yet,
|
||||
// look for active (non-snapshot) VDIs with matching COPY_OF, like the old code did.
|
||||
debug('checkBaseVdis, no snapshot candidates, falling back to legacy active VDI lookup')
|
||||
@@ -137,9 +92,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
* 6.3+ snapshot-based validation: for each snapshot candidate, check whether
|
||||
* the active VDI has diverged from the snapshot. Returns a baseVdisBySourceUuid
|
||||
* map and, when all disks are clean, the targetVmRef to reuse.
|
||||
*
|
||||
* @param {Map<XenApiVdi['id'], import('@vates/types').XenApiVdi>} snapshotCandidates - Snapshot VDIs on the target SR to validate.
|
||||
* @returns {Promise<{ baseVdisBySourceUuid: Map<string, import('@vates/types').XenApiVdi>, targetVmRef: import('@vates/types').XenApiVm['$ref'] | undefined }>}
|
||||
*/
|
||||
async #validateSnapshotCandidates(snapshotCandidates) {
|
||||
const sr = this._sr
|
||||
@@ -148,36 +100,24 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
let canChainToTargetVm = true
|
||||
|
||||
await asyncEach(
|
||||
snapshotCandidates.entries(),
|
||||
async ([sourceUuid, snapshot]) => {
|
||||
snapshotCandidates,
|
||||
async snapshot => {
|
||||
let diffDisk
|
||||
let activeVdi
|
||||
try {
|
||||
activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
|
||||
const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
|
||||
if (userVbds.length !== 1) {
|
||||
canChainToTargetVm = false
|
||||
debug('checkBaseVdis, shared vbd ', { ref: snapshot.$ref, userVbds })
|
||||
// shared vdi ignore / don't chain
|
||||
// shared vdi ignore
|
||||
return
|
||||
}
|
||||
const vm = userVbds[0].$VM
|
||||
|
||||
// a running VM will fail to compute disk exports
|
||||
// also a running VM can be assumed to have changed data
|
||||
if (vm.power_state !== VM_POWER_STATE.HALTED && vm.power_state !== VM_POWER_STATE.SUSPENDED) {
|
||||
canChainToTargetVm = false
|
||||
debug('checkBaseVdis, target vm is not halted or suspended', {
|
||||
ref: snapshot.$ref,
|
||||
userVbds,
|
||||
powerState: vm.power_state,
|
||||
})
|
||||
}
|
||||
// from this disk of from another
|
||||
// skip the costly part, only do the disk chaining
|
||||
if (!canChainToTargetVm) {
|
||||
debug("Can't chain VM anyway , fast return and chain with snapshot")
|
||||
baseVdisBySourceUuid.set(sourceUuid, snapshot)
|
||||
if (!('start' in vm.blocked_operations)) {
|
||||
debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
|
||||
// vm start unlocked
|
||||
// not really an issue since we have check the delta
|
||||
// but it indicates the users played with the blocked operations
|
||||
return
|
||||
}
|
||||
diffDisk = new XapiDiskSource({
|
||||
@@ -187,16 +127,18 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
onlyListChangedBlocks: true,
|
||||
})
|
||||
await diffDisk.init()
|
||||
const sourceUuid = snapshot.other_config?.[COPY_OF]
|
||||
if (diffDisk.getBlockIndexes().length === 0) {
|
||||
debug(' NO CHANGE , source detected ? ', !!sourceUuid)
|
||||
// no block modification since the common snapshot, we can chain VM and disk
|
||||
// the disk is chained with the active to keep the chain linear
|
||||
baseVdisBySourceUuid.set(sourceUuid, activeVdi)
|
||||
if (sourceUuid) {
|
||||
baseVdisBySourceUuid.set(sourceUuid, activeVdi)
|
||||
}
|
||||
// Track the target VM (the replicated VM to update on the next transfer).
|
||||
targetVmRef = vm.$ref
|
||||
} else {
|
||||
debug(' GOT CHANGE, source detected ? ', !!sourceUuid)
|
||||
baseVdisBySourceUuid.set(sourceUuid, snapshot)
|
||||
if (sourceUuid) {
|
||||
baseVdisBySourceUuid.set(sourceUuid, snapshot)
|
||||
}
|
||||
// there are changed block since the snapshot
|
||||
// we can reuse it to transfer a delta, but we will
|
||||
// create a new VM
|
||||
@@ -223,7 +165,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
)
|
||||
|
||||
if (!canChainToTargetVm) {
|
||||
debug('checkBaseVdis,NOT a valid vm target')
|
||||
// if at least one disk has new data, create a new VM
|
||||
// instead of updating it
|
||||
targetVmRef = undefined
|
||||
@@ -231,7 +172,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
debug('checkBaseVdis,got a valid vm target', targetVmRef)
|
||||
}
|
||||
|
||||
debug('checkBaseVdis,base vdis found : ', baseVdisBySourceUuid.size)
|
||||
return { baseVdisBySourceUuid, targetVmRef }
|
||||
}
|
||||
|
||||
@@ -329,10 +269,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
}
|
||||
|
||||
Object.values(backup.vdis).forEach(vdi => {
|
||||
// setVmSnapshotContentKeys sets CONTENT_KEY = the snapshot VDI's own UUID, but
|
||||
// that XenAPI write may not be visible in the xapi cache yet when exportIncrementalVm
|
||||
// reads vdi.other_config. Derive the correct value directly instead of relying on cache.
|
||||
vdi.other_config[CONTENT_KEY] = vdi.uuid
|
||||
vdi.other_config[COPY_OF] = vdi.uuid
|
||||
vdi.other_config[JOB_ID] = job.id
|
||||
vdi.other_config[SCHEDULE_ID] = scheduleId
|
||||
@@ -391,7 +327,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
await xapi.VM_snapshot(targetVmRef, {
|
||||
name_label: `${vm.name_label} - ${job.name} / ${schedule.name} ${formatFilenameDate(timestamp)}`,
|
||||
})
|
||||
await resetVmOtherConfig(xapi, targetVmRef)
|
||||
})
|
||||
|
||||
return {
|
||||
|
||||
@@ -31,6 +31,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
return await Task.run({ properties: { name: 'clean-vm' } }, () => {
|
||||
return this._adapter.cleanVm(this._vmBackupDir, {
|
||||
...options,
|
||||
fixMetadata: true,
|
||||
logInfo: info,
|
||||
logWarn: (message, data) => {
|
||||
warn(message, data)
|
||||
@@ -38,6 +39,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
},
|
||||
lock: false,
|
||||
mergeBlockConcurrency: this._config.mergeBlockConcurrency,
|
||||
removeTmp: true,
|
||||
})
|
||||
})
|
||||
} catch (error) {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
/**
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import assert from 'assert'
|
||||
@@ -10,15 +10,15 @@ import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { basename, dirname } from 'path'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { relativeFromFile, normalize, resolveFromFile } from '@xen-orchestra/fs/path'
|
||||
import { relativeFromFile } from '@xen-orchestra/fs/path'
|
||||
|
||||
// @ts-ignore
|
||||
const { warn } = createLogger('remote-disk:merge')
|
||||
|
||||
/**
|
||||
* @typedef {Object} MergeState
|
||||
* @property {{ uuid?: string }} child
|
||||
* @property {{ uuid?: string }} parent
|
||||
* @property {{ uuid: string }} child
|
||||
* @property {{ uuid: string }} parent
|
||||
* @property { string[] | undefined} chain
|
||||
* @property {number} currentBlock
|
||||
* @property {number} mergedDataSize
|
||||
@@ -82,12 +82,12 @@ export class MergeRemoteDisk {
|
||||
#lastStateWrittenAt
|
||||
|
||||
/**
|
||||
* @type {RemoteHandlerAbstract}
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @param {RemoteHandlerAbstract} handler
|
||||
* @param {FileAccessor} handler
|
||||
* @param {Object} params
|
||||
* @param {Function} [params.onProgress]
|
||||
* @param {Logger | Function} [params.logInfo]
|
||||
@@ -222,7 +222,6 @@ export class MergeRemoteDisk {
|
||||
await this.#mergeBlocks(parentDisk, childDisk)
|
||||
await parentDisk.flushMetadata(childDisk)
|
||||
await parentDisk.mergeMetadata(childDisk)
|
||||
this.#state.diskSize = parentDisk.getSizeOnDisk()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -267,6 +266,8 @@ export class MergeRemoteDisk {
|
||||
|
||||
await this.#writeState()
|
||||
|
||||
this.#state.diskSize = childDisk.getSizeOnDisk()
|
||||
|
||||
this.#onProgress({ total: nBlocks, done: nBlocks })
|
||||
}
|
||||
|
||||
@@ -321,44 +322,4 @@ export class MergeRemoteDisk {
|
||||
|
||||
return { mergedDataSize, finalDiskSize }
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans vdiDir files for interrupted merge state files and resolves their chains
|
||||
*
|
||||
* @param {RemoteHandlerAbstract} handler
|
||||
* @param {string} vdiDir
|
||||
* @param {string[]} files - all file paths in vdiDir (prependDir: true)
|
||||
* @param {Map<string, string>} uuidToPath - diskUuid: normalized disk path
|
||||
* @param {Map<string, string>} childOf - parentPath: child path (from disk headers)
|
||||
* @returns {Promise<Map<string, { stateFilePath: string, chain?: string[] }>>}
|
||||
*/
|
||||
static async findInterruptedMerges(handler, vdiDir, files, uuidToPath, childOf) {
|
||||
const STATE_FILE_RE = /^\.(.+)\.merge\.json$/
|
||||
const result = new Map()
|
||||
|
||||
for (const filePath of files) {
|
||||
const match = STATE_FILE_RE.exec(basename(filePath))
|
||||
if (match === null) continue
|
||||
|
||||
const parentPath = normalize(vdiDir + '/' + match[1])
|
||||
let chain
|
||||
try {
|
||||
const state = JSON.parse(await handler.readFile(filePath))
|
||||
if (Array.isArray(state?.chain)) {
|
||||
chain = state.chain.map((/** @type {string} */ relPath) => normalize(resolveFromFile(filePath, relPath)))
|
||||
} else {
|
||||
// old state file without chain: fall back to UUID lookup then header-based map
|
||||
const childUuid = state?.child?.uuid
|
||||
const childPath = childUuid !== undefined ? uuidToPath.get(childUuid) : childOf.get(parentPath)
|
||||
if (childPath !== undefined) chain = [parentPath, childPath]
|
||||
}
|
||||
} catch (error) {
|
||||
warn("Merge state unreadable, can't restart merging.", error)
|
||||
}
|
||||
|
||||
result.set(parentPath, { stateFilePath: filePath, chain })
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -73,30 +73,6 @@ export class RemoteDisk extends RandomAccessDisk {
|
||||
throw new Error(`getUuid must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getParentUuid() {
|
||||
throw new Error(`getParentUuid must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getParentPath() {
|
||||
throw new Error(`getParentPath must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {object}
|
||||
*/
|
||||
getMetadata() {
|
||||
throw new Error(`getMetadata must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
@@ -112,18 +88,6 @@ export class RemoteDisk extends RandomAccessDisk {
|
||||
return Math.ceil(this.getVirtualSize() / this.getBlockSize())
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if every block present in `other` is also present in this disk.
|
||||
* @param {RemoteDisk} other
|
||||
* @returns {boolean}
|
||||
*/
|
||||
containsAllDataOf(other) {
|
||||
for (const blockIndex of other.getBlockIndexes()) {
|
||||
if (!this.hasBlock(blockIndex)) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
@@ -241,34 +205,11 @@ export class RemoteDisk extends RandomAccessDisk {
|
||||
/**
|
||||
* Abstract
|
||||
* Deletes alias/disk/disks
|
||||
* @param {Object} options
|
||||
*/
|
||||
async unlink({ force = false } = {}) {
|
||||
async unlink() {
|
||||
throw new Error(`unlink must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the integrity of the disk's external reference (e.g. alias target).
|
||||
* @param {Object} [opts]
|
||||
* @param {boolean} [opts.remove]
|
||||
* @param {Function} [opts.logWarn]
|
||||
* @param {Function} [opts.logInfo]
|
||||
* @returns {Promise<string | undefined>}
|
||||
*/
|
||||
async clean(opts = {}) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all file paths within dir that this disk claims.
|
||||
*
|
||||
* @param {string} dir
|
||||
* @returns {Promise<string[]>}
|
||||
*/
|
||||
async listAssociatedFiles(dir) {
|
||||
throw new Error(`listAssociatedFiles must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
@@ -3,9 +3,9 @@
|
||||
/**
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
* @typedef {import('vhd-lib/_createFooterHeader.js').VhdFooter} VhdFooter
|
||||
* @typedef {import('vhd-lib/_createFooterHeader').VhdFooter} VhdFooter
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/fs').RemoteHandlerAbstract} RemoteHandlerAbstract
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*
|
||||
|
||||
*/
|
||||
@@ -17,7 +17,6 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
import { stringify } from 'uuid'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
|
||||
import { normalize } from '@xen-orchestra/fs/path'
|
||||
|
||||
export class RemoteVhdDisk extends RemoteDisk {
|
||||
/**
|
||||
@@ -26,7 +25,7 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {RemoteHandlerAbstract}
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
@@ -57,7 +56,7 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {RemoteHandlerAbstract} params.handler
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
@@ -70,7 +69,6 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
/**
|
||||
* @param {Object} [options]
|
||||
* @param {boolean} [options.force=false]
|
||||
* @param {boolean} [options.ignoreBlockIndexes=false]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options = {}) {
|
||||
@@ -87,9 +85,7 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
}
|
||||
|
||||
this.#dispose = dispose
|
||||
if (!options.ignoreBlockIndexes) {
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
}
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
} catch (error) {
|
||||
await this.close()
|
||||
@@ -176,20 +172,6 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
return stringify(this.#vhd.header.parentUuid)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the parent path
|
||||
*
|
||||
* @returns {string}
|
||||
*/
|
||||
getParentPath() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getParentPath of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
return normalize(join(dirname(this.#path), parentPath))
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
@@ -236,7 +218,7 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
}
|
||||
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = normalize(join(dirname(this.#path), parentPath))
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
|
||||
if (!parentPath) {
|
||||
throw new Error(`disk ${this.#path} doesn't have parents`)
|
||||
@@ -245,13 +227,6 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
const parent = new RemoteVhdDisk({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
/**
|
||||
* Parent of a RemoteVhdDisk is a RemoteVhdDisk
|
||||
* @returns {Promise<RemoteVhdDisk>}
|
||||
*/
|
||||
async openParent() {
|
||||
return /** @type {Promise<RemoteVhdDisk>} */ (super.openParent())
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a full block into this VHD.
|
||||
@@ -398,7 +373,7 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async mergeMetadata(childDisk) {
|
||||
@@ -441,9 +416,10 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
if (isVhdAlias(newPath)) {
|
||||
const dataPath = await resolveVhdAlias(this.#handler, this.#path)
|
||||
|
||||
await this.#handler.unlink(newPath)
|
||||
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
|
||||
await this.#handler.unlink(this.#path)
|
||||
await this.#handler.unlink(newPath)
|
||||
|
||||
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
|
||||
|
||||
this.#path = newPath
|
||||
} else {
|
||||
@@ -463,59 +439,31 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
|
||||
/**
|
||||
* Deletes disk
|
||||
* @param {Object} options
|
||||
*/
|
||||
async unlink({ force = false } = {}) {
|
||||
async unlink() {
|
||||
if (this.#vhd === undefined) {
|
||||
if (force) {
|
||||
let resolved = this.#path
|
||||
try {
|
||||
resolved = await resolveVhdAlias(this.#handler, this.#path)
|
||||
} catch (err) {
|
||||
// broken vhd directory must be unlinkable
|
||||
if (err.code !== 'EISDIR') {
|
||||
throw err
|
||||
}
|
||||
// warn('Deleting directly a VhdDirectory', { this.#path, err })
|
||||
}
|
||||
try {
|
||||
await this.#handler.unlink(resolved)
|
||||
} catch (err) {
|
||||
if (err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(resolved)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
// also delete the alias file
|
||||
if (this.#path !== resolved) {
|
||||
await this.#handler.unlink(this.#path)
|
||||
}
|
||||
} else {
|
||||
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
|
||||
}
|
||||
} else {
|
||||
await this.close()
|
||||
|
||||
if (isVhdAlias(this.#path)) {
|
||||
try {
|
||||
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.close()
|
||||
|
||||
if (isVhdAlias(this.#path)) {
|
||||
try {
|
||||
await this.#handler.unlink(this.#path)
|
||||
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(this.#path).catch(() => {})
|
||||
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#handler.unlink(this.#path)
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(this.#path).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -529,118 +477,4 @@ export class RemoteVhdDisk extends RemoteDisk {
|
||||
|
||||
return this.#vhd instanceof VhdDirectory
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all file paths within dir that this disk claims.
|
||||
* For a plain VHD or VHD directory: [path]
|
||||
* For a VHD alias: [aliasPath, resolvedDataPath]
|
||||
* If the alias target cannot be resolved, only the alias path is returned.
|
||||
*
|
||||
* @param {string} dir
|
||||
* @returns {Promise<string[]>}
|
||||
*/
|
||||
async listAssociatedFiles(dir) {
|
||||
const prefix = normalize(dir.endsWith('/') ? dir : dir + '/')
|
||||
const isInDir = /** @param {string} p */ p => p === dir || p.startsWith(prefix)
|
||||
|
||||
const files = []
|
||||
if (isInDir(this.#path)) {
|
||||
files.push(this.#path)
|
||||
}
|
||||
|
||||
if (isVhdAlias(this.#path)) {
|
||||
try {
|
||||
const resolved = await resolveVhdAlias(this.#handler, this.#path)
|
||||
if (isInDir(resolved)) {
|
||||
files.push(resolved)
|
||||
}
|
||||
} catch {
|
||||
// broken alias, no data file to claim
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the integrity of this disk's alias reference.
|
||||
* Only meaningful for alias files (.alias.vhd); no-op for plain VHDs.
|
||||
* Returns the resolved target path when the alias is valid, undefined otherwise.
|
||||
*
|
||||
* @param {Object} [opts]
|
||||
* @param {boolean} [opts.remove]
|
||||
* @param {Function} [opts.logWarn]
|
||||
* @param {Function} [opts.logInfo]
|
||||
* @returns {Promise<string | undefined>}
|
||||
*/
|
||||
async clean({ remove = false, logWarn = () => {}, logInfo = () => {} } = {}) {
|
||||
if (!isVhdAlias(this.#path)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
let target
|
||||
try {
|
||||
target = await resolveVhdAlias(this.#handler, this.#path)
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
logWarn('missing target of alias', { alias: this.#path })
|
||||
if (remove) {
|
||||
logInfo('removing alias with missing target', { alias: this.#path })
|
||||
await this.#handler.unlink(this.#path)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
if (err.code === 'EISDIR') {
|
||||
logWarn('alias is a vhd directory', { alias: this.#path })
|
||||
if (remove) {
|
||||
logInfo('removing vhd directory named as alias', { alias: this.#path })
|
||||
await VhdAbstract.unlink(this.#handler, this.#path)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
logWarn('unhandled error while checking alias', { alias: this.#path, err })
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (target === '') {
|
||||
logWarn('empty target for alias', { alias: this.#path })
|
||||
if (remove) {
|
||||
logInfo('removing alias with empty target', { alias: this.#path })
|
||||
await this.#handler.unlink(this.#path)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (!isVhdAlias(target) && !target.endsWith('.vhd')) {
|
||||
logWarn('alias references non VHD target', { alias: this.#path, target })
|
||||
if (remove) {
|
||||
logInfo('removing alias and non VHD target', { alias: this.#path, target })
|
||||
await this.#handler.unlink(target)
|
||||
await this.#handler.unlink(this.#path)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
try {
|
||||
const { dispose } = await openVhd(this.#handler, target)
|
||||
try {
|
||||
await dispose()
|
||||
} catch (_) {
|
||||
// errors during dispose should not trigger deletion
|
||||
}
|
||||
return target
|
||||
} catch (error) {
|
||||
logWarn('missing or broken alias target', { alias: this.#path, target, error })
|
||||
if (remove) {
|
||||
try {
|
||||
await VhdAbstract.unlink(this.#handler, this.#path)
|
||||
} catch (/** @type {any} */ err) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
logWarn('error deleting broken alias', { alias: this.#path, target, err })
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import { RemoteDisk } from './RemoteDisk.mjs'
|
||||
@@ -36,6 +37,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {RemoteVhdDisk[]} params.disks
|
||||
*/
|
||||
constructor({ disks }) {
|
||||
@@ -246,7 +248,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
@@ -254,7 +256,7 @@ export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
mergeMetadata(childDisk) {
|
||||
@@ -280,11 +282,10 @@ export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
|
||||
/**
|
||||
* Deletes all the disks
|
||||
* @param {Object} options
|
||||
*/
|
||||
async unlink({ force = false } = {}) {
|
||||
async unlink() {
|
||||
for (const disk of this.#disks) {
|
||||
await disk.unlink({ force })
|
||||
await disk.unlink()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,16 +301,4 @@ export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the integrity of the disk's external reference (e.g. alias target).
|
||||
* @param {Object} [opts]
|
||||
* @param {boolean} [opts.remove]
|
||||
* @param {Function} [opts.logWarn]
|
||||
* @param {Function} [opts.logInfo]
|
||||
* @returns {Promise<string | undefined>}
|
||||
*/
|
||||
async clean(opts = {}) {
|
||||
throw new Error(`Can't clean a disk chain`)
|
||||
}
|
||||
}
|
||||
49
@xen-orchestra/backups/disks/index.mjs
Normal file
49
@xen-orchestra/backups/disks/index.mjs
Normal file
@@ -0,0 +1,49 @@
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
export { RemoteDisk } from './RemoteDisk.mjs'
|
||||
export { openDiskChain } from './openDiskChain.mjs'
|
||||
|
||||
/**
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
export async function openDisk({ handler, path }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
return disk
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @returns {Promise<Disposable<RemoteDisk>>}
|
||||
*/
|
||||
export async function openDisposableDisk({ handler, path }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
return {
|
||||
value: disk,
|
||||
dispose: () => disk.close(),
|
||||
}
|
||||
}
|
||||
|
||||
const DISK_EXTENSIONS = ['.vhd']
|
||||
|
||||
/**
|
||||
* Returns true if the path points to a supported disk format.
|
||||
*
|
||||
* @param {FileAccessor} _handler - Remote file handler (reserved for future use)
|
||||
* @param {string} path - Path to check
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isDisk(_handler, path) {
|
||||
return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
|
||||
}
|
||||
40
@xen-orchestra/backups/disks/openDiskChain.mjs
Normal file
40
@xen-orchestra/backups/disks/openDiskChain.mjs
Normal file
@@ -0,0 +1,40 @@
|
||||
// @ts-check
|
||||
/**
|
||||
*
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
import { DiskChain } from '@xen-orchestra/disk-transform'
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
import { defer } from 'golike-defer'
|
||||
/**
|
||||
* @param {any} $defer
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @param {string | undefined} params.until
|
||||
*/
|
||||
async function _openDiskChain($defer, { handler, path, until }) {
|
||||
let disk
|
||||
/**
|
||||
* @type {Array<RemoteDisk>}
|
||||
*/
|
||||
const disks = []
|
||||
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
|
||||
await disk.init()
|
||||
disks.push(disk)
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent()
|
||||
if (disk.getPath() === until) {
|
||||
break
|
||||
}
|
||||
disks.unshift(disk)
|
||||
}
|
||||
// the root disk
|
||||
return new DiskChain({ disks })
|
||||
}
|
||||
|
||||
export const openDiskChain = defer(_openDiskChain)
|
||||
@@ -8,7 +8,7 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.72.1",
|
||||
"version": "0.72.0",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
},
|
||||
@@ -30,14 +30,12 @@
|
||||
"@vates/nbd-client": "^3.4.0",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@vates/task": "^0.7.0",
|
||||
"@vates/types": "^1.24.0",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/disk-transform": "^1.2.3",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"@xen-orchestra/qcow2": "^1.3.0",
|
||||
"@xen-orchestra/template": "^0.1.1",
|
||||
"@xen-orchestra/backup-archive": "^1.0.0",
|
||||
"app-conf": "^3.0.0",
|
||||
"compare-versions": "^6.0.0",
|
||||
"d3-time-format": "^4.1.0",
|
||||
@@ -63,8 +61,7 @@
|
||||
"fs-extra": "^11.1.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"sinon": "^18.0.0",
|
||||
"tmp": "^0.2.1",
|
||||
"typescript": "^5.9.3"
|
||||
"tmp": "^0.2.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^8.7.2"
|
||||
@@ -75,6 +72,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"exports": {
|
||||
"./disks": "./disks/index.mjs",
|
||||
"./*": "./*"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
export namespace VHDFOOTER {
|
||||
let cookie: string
|
||||
let features: number
|
||||
let fileFormatVersion: number
|
||||
let dataOffset: number
|
||||
let timestamp: number
|
||||
let creatorApplication: string
|
||||
let creatorVersion: number
|
||||
let creatorHostOs: number
|
||||
let originalSize: number
|
||||
let currentSize: number
|
||||
namespace diskGeometry {
|
||||
let cylinders: number
|
||||
let heads: number
|
||||
let sectorsPerTrackCylinder: number
|
||||
}
|
||||
let diskType: number
|
||||
let checksum: number
|
||||
let uuid: Buffer<ArrayBuffer>
|
||||
let saved: string
|
||||
let hidden: string
|
||||
let reserved: string
|
||||
}
|
||||
export namespace VHDHEADER {
|
||||
let cookie_1: string
|
||||
export { cookie_1 as cookie }
|
||||
let dataOffset_1: any
|
||||
export { dataOffset_1 as dataOffset }
|
||||
export let tableOffset: number
|
||||
export let headerVersion: number
|
||||
export let maxTableEntries: number
|
||||
export let blockSize: number
|
||||
let checksum_1: number
|
||||
export { checksum_1 as checksum }
|
||||
export let parentUuid: any
|
||||
export let parentTimestamp: number
|
||||
export let reserved1: number
|
||||
export let parentUnicodeName: string
|
||||
export let parentLocatorEntry: {
|
||||
platformCode: number
|
||||
platformDataSpace: number
|
||||
platformDataLength: number
|
||||
reserved: number
|
||||
platformDataOffset: number
|
||||
}[]
|
||||
export let reserved2: string
|
||||
}
|
||||
@@ -20,8 +20,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@xen-orchestra/backups": "^0.72.1",
|
||||
"@xen-orchestra/backup-archive": "^1.0.0",
|
||||
"@xen-orchestra/backups": "^0.72.0",
|
||||
"@xen-orchestra/disk-transform": "^1.2.3",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"@xen-orchestra/qcow2": "^1.3.0",
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import { openDisposableDisk, RemoteDisk } from '@xen-orchestra/backup-archive/disks'
|
||||
import { openDisposableDisk } from '@xen-orchestra/backups/disks'
|
||||
import { formatBytes } from '../utils.mjs'
|
||||
|
||||
export async function infoCommand(handlerUrl: string, diskPath: string, _extraArgs: string[]): Promise<void> {
|
||||
await Disposable.use(getSyncedHandler({ url: handlerUrl }), async handler => {
|
||||
await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async (disk: RemoteDisk) => {
|
||||
await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async disk => {
|
||||
const isDifferencing = disk.isDifferencing()
|
||||
const virtualSize = disk.getVirtualSize()
|
||||
const blockSize = disk.getBlockSize()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import { isDisk, openDisposableDisk, RemoteDisk } from '@xen-orchestra/backup-archive/disks'
|
||||
import { isDisk, openDisposableDisk } from '@xen-orchestra/backups/disks'
|
||||
import { formatBytes, renderTable } from '../utils.mjs'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
const HEADERS = ['File', 'UID', 'Size on disk', 'Virtual size', 'Differencing', 'Parent UID']
|
||||
@@ -87,21 +87,18 @@ export async function listCommand(handlerUrl: string, dirPath: string, _extraArg
|
||||
async (diskPath: string) => {
|
||||
const filename = diskPath.slice(diskPath.lastIndexOf('/') + 1)
|
||||
try {
|
||||
const disk = await Disposable.use(
|
||||
openDisposableDisk({ handler, path: diskPath }),
|
||||
async (disk: RemoteDisk) => {
|
||||
const differencing = disk.isDifferencing()
|
||||
return {
|
||||
ok: true as const,
|
||||
filename,
|
||||
uid: disk.getUuid(),
|
||||
sizeOnDisk: formatBytes(disk.getSizeOnDisk()),
|
||||
virtualSize: formatBytes(disk.getVirtualSize()),
|
||||
differencing,
|
||||
parentUid: differencing ? disk.getParentUuid() : null,
|
||||
}
|
||||
const disk = await Disposable.use(openDisposableDisk({ handler, path: diskPath }), async disk => {
|
||||
const differencing = disk.isDifferencing()
|
||||
return {
|
||||
ok: true as const,
|
||||
filename,
|
||||
uid: disk.getUuid(),
|
||||
sizeOnDisk: formatBytes(disk.getSizeOnDisk()),
|
||||
virtualSize: formatBytes(disk.getVirtualSize()),
|
||||
differencing,
|
||||
parentUid: differencing ? disk.getParentUuid() : null,
|
||||
}
|
||||
)
|
||||
})
|
||||
disks.push(disk)
|
||||
} catch (err) {
|
||||
disks.push({ ok: false as const, filename, error: err instanceof Error ? err.message : String(err) })
|
||||
|
||||
@@ -3,7 +3,7 @@ import { Readable } from 'node:stream'
|
||||
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { getSyncedHandler, type RemoteHandler } from '@xen-orchestra/fs'
|
||||
import { openDisposableDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
|
||||
import { openDisposableDisk, openDiskChain } from '@xen-orchestra/backups/disks'
|
||||
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { toQcow2Stream } from '@xen-orchestra/qcow2'
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ declare module 'vhd-lib/disk-consumer/index.mjs' {
|
||||
export function toVhdStream(disk: RandomAccessDisk): Promise<Readable>
|
||||
}
|
||||
|
||||
declare module '@xen-orchestra/backup-archive/disks/openDiskChain.mjs' {
|
||||
declare module '@xen-orchestra/backups/disks/openDiskChain.mjs' {
|
||||
import type { RemoteHandler } from '@xen-orchestra/fs'
|
||||
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
|
||||
@@ -45,7 +45,7 @@ declare module '@xen-orchestra/backup-archive/disks/openDiskChain.mjs' {
|
||||
}): Promise<RandomAccessDisk>
|
||||
}
|
||||
|
||||
declare module '@xen-orchestra/backup-archive/disks' {
|
||||
declare module '@xen-orchestra/backups/disks' {
|
||||
import type { RemoteHandler } from '@xen-orchestra/fs'
|
||||
import type { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user