mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-16 03:51:39 -06:00
Compare commits
116 Commits
feat_rolli
...
debug_back
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eed3d72f75 | ||
|
|
cb85e44aec | ||
|
|
29b7a67dc3 | ||
|
|
353678e754 | ||
|
|
a313986694 | ||
|
|
983ac2d2c9 | ||
|
|
4e50858526 | ||
|
|
c497ae3d38 | ||
|
|
9d9b5d5e33 | ||
|
|
5834070e77 | ||
|
|
5c5f27f92b | ||
|
|
8fc04bc03b | ||
|
|
87135c7469 | ||
|
|
fb030a590e | ||
|
|
0be2392519 | ||
|
|
546d42427c | ||
|
|
fc86fda13d | ||
|
|
b44bed6d10 | ||
|
|
4e66889233 | ||
|
|
10bb1741e7 | ||
|
|
a0f890f301 | ||
|
|
839a484017 | ||
|
|
dc9e029bdb | ||
|
|
f203b1f1e1 | ||
|
|
7399a9fec5 | ||
|
|
c69b2336f6 | ||
|
|
80ab29de29 | ||
|
|
1449a8482e | ||
|
|
ec9806e081 | ||
|
|
c6c31524e1 | ||
|
|
49fce9b61f | ||
|
|
f52a35c093 | ||
|
|
9d74585bd4 | ||
|
|
12f4360100 | ||
|
|
771ce8c6d5 | ||
|
|
41c9a6c7c2 | ||
|
|
7e08a016bd | ||
|
|
91c5d98489 | ||
|
|
b895f48a4f | ||
|
|
d68d878747 | ||
|
|
5155a9d5f1 | ||
|
|
7437427fae | ||
|
|
a16b4cb540 | ||
|
|
21508c97bc | ||
|
|
e8ff60c8bc | ||
|
|
153efccb85 | ||
|
|
a99e1aa72d | ||
|
|
dd5f0cdaa9 | ||
|
|
ea142b50a4 | ||
|
|
5b5a0a72cc | ||
|
|
84a16ef85f | ||
|
|
b983e6d4fa | ||
|
|
a0e771f09c | ||
|
|
9bd91aea43 | ||
|
|
0a042d2f54 | ||
|
|
c610951450 | ||
|
|
a831ac7b21 | ||
|
|
d0f71d4ec7 | ||
|
|
c1a9bc82fb | ||
|
|
601cc3e9fd | ||
|
|
79d1f42dd7 | ||
|
|
64c366838a | ||
|
|
85b3162be6 | ||
|
|
eb020e250b | ||
|
|
56142085e5 | ||
|
|
2838bde875 | ||
|
|
fa110ed9c9 | ||
|
|
d3187c06b5 | ||
|
|
2f41ccffe8 | ||
|
|
a9e45fcb8e | ||
|
|
1e69c25ba5 | ||
|
|
9f852365fc | ||
|
|
05824cb8a6 | ||
|
|
638ee77aff | ||
|
|
08a560a8da | ||
|
|
e77a0e43f1 | ||
|
|
8069adf26d | ||
|
|
1241977599 | ||
|
|
4ca314fbc7 | ||
|
|
13ab1cce86 | ||
|
|
9f7797fdda | ||
|
|
21767145bd | ||
|
|
6805e16108 | ||
|
|
ecd49e69a7 | ||
|
|
a5f5a0273e | ||
|
|
ddc9bf3211 | ||
|
|
75db97f9ea | ||
|
|
07c1c02f03 | ||
|
|
dbe8e0ce26 | ||
|
|
82cd861479 | ||
|
|
32f8c1a25f | ||
|
|
9e7813319a | ||
|
|
caba00a2b9 | ||
|
|
927561123a | ||
|
|
097c2bd1d9 | ||
|
|
5288b6d848 | ||
|
|
263059be22 | ||
|
|
c83df8a342 | ||
|
|
4f959e2d22 | ||
|
|
ee6df8c979 | ||
|
|
0a0ae5dc7e | ||
|
|
58b6215c41 | ||
|
|
0e56c5e2bd | ||
|
|
0b44b1b52b | ||
|
|
4c7b8b194f | ||
|
|
ac2ed8880a | ||
|
|
6fc30c899d | ||
|
|
05e1298eec | ||
|
|
d20561fadc | ||
|
|
53a3e2a70f | ||
|
|
2c90eb5ba8 | ||
|
|
c3e78ebdfd | ||
|
|
264b93d502 | ||
|
|
d0abf8810c | ||
|
|
bdd9b26651 | ||
|
|
66d5014755 |
21
.eslintrc.js
21
.eslintrc.js
@@ -233,7 +233,26 @@ module.exports = {
|
||||
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
|
||||
'@intlify/vue-i18n/no-missing-keys-in-other-locales': [
|
||||
'error',
|
||||
{ ignoreLocales: ['de', 'fa', 'cs', 'es', 'sv', 'it', 'ru', 'uk', 'nl', 'pt-BR', 'da', 'ko', 'nb-NO', 'ja'] },
|
||||
{
|
||||
ignoreLocales: [
|
||||
'de',
|
||||
'fa',
|
||||
'cs',
|
||||
'es',
|
||||
'sv',
|
||||
'it',
|
||||
'ru',
|
||||
'uk',
|
||||
'nl',
|
||||
'pt-BR',
|
||||
'da',
|
||||
'ko',
|
||||
'nb-NO',
|
||||
'ja',
|
||||
'fi',
|
||||
'pl',
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
@@ -19,12 +19,12 @@
|
||||
},
|
||||
"main": "./index.mjs",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/fs": "^4.6.5",
|
||||
"@xen-orchestra/fs": "^4.6.6",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"fuse-native": "^2.2.6",
|
||||
"lru-cache": "^7.14.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"vhd-lib": "^4.14.6"
|
||||
"vhd-lib": "^4.14.7"
|
||||
},
|
||||
"bin": {
|
||||
"xo-fuse-vhd": "./cli.mjs"
|
||||
|
||||
@@ -127,6 +127,6 @@ export class NbdDisk extends RandomAccessDisk {
|
||||
}
|
||||
const blockStart = index * this.getBlockSize()
|
||||
const blockEnd = (index + 1) * this.getBlockSize()
|
||||
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && blockEnd >= offset)
|
||||
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && offset < blockEnd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "3.2.2",
|
||||
"version": "3.2.3",
|
||||
"engines": {
|
||||
"node": ">=14.0"
|
||||
},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@vates/types",
|
||||
"private": false,
|
||||
"version": "1.16.0",
|
||||
"version": "1.19.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": {
|
||||
@@ -9,6 +9,7 @@
|
||||
"types": "./dist/index.d.mts"
|
||||
},
|
||||
"./xo": "./dist/xo.d.mts",
|
||||
"./xo-app": "./dist/xo-app.d.mts",
|
||||
"./xen-api": "./dist/xen-api.d.mts",
|
||||
"./common": {
|
||||
"default": "./dist/common.mjs",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export * from './common.mjs'
|
||||
export * from './xen-api.mjs'
|
||||
export * from './xo.mjs'
|
||||
export * from './xo-app.mjs'
|
||||
export * from './lib/xen-orchestra-xapi.mjs'
|
||||
export * from './lib/complex-matcher.mjs'
|
||||
|
||||
@@ -1,28 +1,33 @@
|
||||
import {
|
||||
WrappedXenApiRecord,
|
||||
XenApiHost,
|
||||
XenApiNetwork,
|
||||
XenApiNetworkWrapped,
|
||||
XenApiRecord,
|
||||
XenApiSr,
|
||||
XenApiTask,
|
||||
XenApiVbd,
|
||||
XenApiVdi,
|
||||
XenApiVif,
|
||||
XenApiVm,
|
||||
XenApiVmWrapped,
|
||||
XenApiVtpm,
|
||||
} from '../xen-api.mjs'
|
||||
import type { OPAQUE_REF_NULL, SUPPORTED_VDI_FORMAT, VBD_MODE, VBD_TYPE, VIF_LOCKING_MODE } from '../common.mjs'
|
||||
import type { PassThrough, Readable } from 'node:stream'
|
||||
import type {
|
||||
XoGpuGroup,
|
||||
XoVgpuType,
|
||||
XoHost,
|
||||
XoNetwork,
|
||||
XoPif,
|
||||
XoSr,
|
||||
XoUser,
|
||||
XoVdi,
|
||||
XoVgpuType,
|
||||
XoVm,
|
||||
XoVmTemplate,
|
||||
XoVif,
|
||||
} from '../xo.mjs'
|
||||
import type { SUPPORTED_VDI_FORMAT } from '../common.mjs'
|
||||
|
||||
export type XcpPatches = {
|
||||
changelog?: {
|
||||
@@ -57,6 +62,7 @@ export interface Xapi {
|
||||
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
|
||||
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
|
||||
getField<T extends XenApiRecord, K extends keyof T>(
|
||||
type: Extract<WrappedXenApiRecord, T>['$type'],
|
||||
ref: T['$ref'],
|
||||
@@ -79,7 +85,21 @@ export interface Xapi {
|
||||
}
|
||||
): Promise<XenApiNetworkWrapped>
|
||||
deleteNetwork(id: XoNetwork['id']): Promise<void>
|
||||
deleteVif(vifId: XoVif['id']): Promise<void>
|
||||
exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
|
||||
migrateVm(
|
||||
vmId: XoVm['id'],
|
||||
hostXapi: Xapi,
|
||||
hostId: XoHost['id'],
|
||||
opts?: {
|
||||
force?: boolean
|
||||
mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
|
||||
mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
|
||||
migrationNetworkId?: XoNetwork['id']
|
||||
sr?: XoSr['id']
|
||||
bypassAssert?: boolean
|
||||
}
|
||||
): Promise<void>
|
||||
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
|
||||
pool_emergencyShutdown(): Promise<void>
|
||||
resumeVm(id: XoVm['id']): Promise<void>
|
||||
@@ -172,11 +192,32 @@ export interface Xapi {
|
||||
gpuGroup?: XoGpuGroup['id']
|
||||
copyHostBiosStrings?: boolean
|
||||
hvmBootFirmware?: 'uefi' | 'bios'
|
||||
secureBoot?: boolean
|
||||
},
|
||||
checkLimits?: boolean,
|
||||
creatorId?: XoUser['id'],
|
||||
opts?: { destroyAllVifs: boolean }
|
||||
): Promise<XenApiVmWrapped>
|
||||
moveVdi(
|
||||
vdiId: XenApiVdi['$ref'] | XoVdi['id'],
|
||||
srId: XenApiSr['$ref'] | XoSr['id'],
|
||||
opts?: { _failOnCbtError?: boolean }
|
||||
): Promise<XenApiVdi>
|
||||
VBD_create(params: {
|
||||
bootable?: boolean
|
||||
empty?: boolean
|
||||
mode?: VBD_MODE
|
||||
other_config?: Record<string, string>
|
||||
qos_algorithm_params?: Record<string, string>
|
||||
qos_algorithm_type?: string
|
||||
type?: VBD_TYPE
|
||||
unpluggable?: boolean
|
||||
userdevice?: string
|
||||
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
|
||||
VM: XenApiVm['$ref']
|
||||
}): Promise<XenApiVbd['$ref']>
|
||||
VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
|
||||
VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
|
||||
VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
|
||||
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
|
||||
VDI_exportContent(
|
||||
@@ -188,6 +229,24 @@ export interface Xapi {
|
||||
stream: Readable,
|
||||
opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
|
||||
): Promise<void>
|
||||
VIF_create(
|
||||
options: {
|
||||
currently_attached?: boolean
|
||||
device?: string
|
||||
ipv4_allowed?: string[]
|
||||
ipv6_allowed?: string[]
|
||||
locking_mode?: VIF_LOCKING_MODE
|
||||
MTU?: number
|
||||
network: XenApiNetwork['$ref']
|
||||
other_config?: Record<string, string>
|
||||
qos_algorithm_params?: Record<string, string>
|
||||
qos_algorithm_type?: string
|
||||
VM: XenApiVm['$ref']
|
||||
},
|
||||
extraOptions?: {
|
||||
MAC?: string
|
||||
}
|
||||
): Promise<XenApiVif['$ref']>
|
||||
VM_createCloudInitConfig(
|
||||
vmRef: XenApiVm['$ref'],
|
||||
cloudConfig: string,
|
||||
@@ -207,4 +266,5 @@ export interface Xapi {
|
||||
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
|
||||
): Promise<{ body: Readable }>
|
||||
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
|
||||
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
|
||||
}
|
||||
|
||||
@@ -1008,6 +1008,7 @@ export interface XenApiMessage {
|
||||
timestamp: string
|
||||
uuid: string
|
||||
}
|
||||
export type XenApiMessageWrapped = WrapperXenApi<XenApiMessage, 'message'>
|
||||
|
||||
export interface XenApiSecret {
|
||||
$ref: Branded<'secret'>
|
||||
|
||||
257
@vates/types/src/xo-app.mts
Normal file
257
@vates/types/src/xo-app.mts
Normal file
@@ -0,0 +1,257 @@
|
||||
import { EventEmitter } from 'node:stream'
|
||||
import type {
|
||||
AnyXoBackupJob,
|
||||
AnyXoJob,
|
||||
AnyXoLog,
|
||||
XapiXoRecord,
|
||||
XoAuthenticationToken,
|
||||
XoBackupRepository,
|
||||
XoConfigBackupArchive,
|
||||
XoGroup,
|
||||
XoHost,
|
||||
XoPool,
|
||||
XoPoolBackupArchive,
|
||||
XoProxy,
|
||||
XoSchedule,
|
||||
XoServer,
|
||||
XoTask,
|
||||
XoUser,
|
||||
XoVm,
|
||||
XoVmBackupArchive,
|
||||
} from './xo.mjs'
|
||||
import { VatesTask } from './lib/vates-task.mjs'
|
||||
import {
|
||||
Xapi,
|
||||
XapiHostStats,
|
||||
XapiPoolStats,
|
||||
XapiStatsGranularity,
|
||||
XapiVmStats,
|
||||
XenApiGpuGroupWrapped,
|
||||
XenApiHostWrapped,
|
||||
XenApiMessageWrapped,
|
||||
XenApiNetworkWrapped,
|
||||
XenApiPbdWrapped,
|
||||
XenApiPciWrapped,
|
||||
XenApiPgpuWrapped,
|
||||
XenApiPifWrapped,
|
||||
XenApiPoolWrapped,
|
||||
XenApiSmWrapped,
|
||||
XenApiSrWrapped,
|
||||
XenApiVbdWrapped,
|
||||
XenApiVdiWrapped,
|
||||
XenApiVgpuTypeWrapped,
|
||||
XenApiVgpuWrapped,
|
||||
XenApiVifWrapped,
|
||||
XenApiVmWrapped,
|
||||
XenApiVtpmWrapped,
|
||||
} from './index.mjs'
|
||||
|
||||
export type XapiConnection = Xapi & {
|
||||
status: string
|
||||
pool?: { uuid: string }
|
||||
sessionId: string
|
||||
_url?: { protocol: string; hostname: string; port?: string }
|
||||
}
|
||||
|
||||
type XapiRecordByXapiXoRecord = {
|
||||
gpuGroup: XenApiGpuGroupWrapped
|
||||
host: XenApiHostWrapped
|
||||
message: XenApiMessageWrapped
|
||||
network: XenApiNetworkWrapped
|
||||
PBD: XenApiPbdWrapped
|
||||
PCI: XenApiPciWrapped
|
||||
PGPU: XenApiPgpuWrapped
|
||||
PIF: XenApiPifWrapped
|
||||
pool: XenApiPoolWrapped
|
||||
SR: XenApiSrWrapped
|
||||
SM: XenApiSmWrapped
|
||||
VBD: XenApiVbdWrapped
|
||||
VDI: XenApiVdiWrapped
|
||||
'VDI-snapshot': XenApiVdiWrapped
|
||||
'VDI-unmanaged': XenApiVdiWrapped
|
||||
vgpu: XenApiVgpuWrapped
|
||||
vgpuType: XenApiVgpuTypeWrapped
|
||||
VIF: XenApiVifWrapped
|
||||
VM: XenApiVmWrapped
|
||||
'VM-controller': XenApiVmWrapped
|
||||
'VM-snapshot': XenApiVmWrapped
|
||||
'VM-template': XenApiVmWrapped
|
||||
VTPM: XenApiVtpmWrapped
|
||||
}
|
||||
export type XoApp = {
|
||||
config: {
|
||||
get<T = string>(path: string): T
|
||||
getOptional(path: string): Record<string, string> | undefined
|
||||
getOptionalDuration(path: string): number | undefined
|
||||
getGuiRoutes(): Promise<{
|
||||
default: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
v5?: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
v6?: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
[key: string]:
|
||||
| undefined
|
||||
| {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
}>
|
||||
}
|
||||
|
||||
objects: EventEmitter & {
|
||||
allIndexes: {
|
||||
type: {
|
||||
getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks: EventEmitter & {
|
||||
abort(id: XoTask['id'], reason?: string): Promise<void>
|
||||
clearLogs(): Promise<void>
|
||||
create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
|
||||
deleteLog(id: XoTask['id']): Promise<void>
|
||||
get(id: XoTask['id']): Promise<XoTask>
|
||||
list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
|
||||
watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
|
||||
}
|
||||
apiContext: {
|
||||
user?: XoUser
|
||||
permission?: XoUser['permission'] | 'none' | null
|
||||
}
|
||||
|
||||
// methods ------------
|
||||
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
|
||||
authenticateUser: (
|
||||
credentials: { token?: string; username?: string; password?: string },
|
||||
userData?: { ip?: string },
|
||||
opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
|
||||
) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
|
||||
/* Throw if no authorization */
|
||||
checkFeatureAuthorization(featureCode: string): Promise<void>
|
||||
/* connect a server (XCP-ng/XenServer) */
|
||||
connectXenServer(id: XoServer['id']): Promise<void>
|
||||
createAuthenticationToken(opts: {
|
||||
client?: {
|
||||
id?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
description?: string
|
||||
expiresIn?: string | number
|
||||
userId: XoUser['id']
|
||||
}): Promise<XoAuthenticationToken>
|
||||
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
|
||||
deleteGroup(id: XoGroup['id']): Promise<void>
|
||||
deleteUser(id: XoUser['id']): Promise<void>
|
||||
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
|
||||
/* disconnect a server (XCP-ng/XenServer) */
|
||||
disconnectXenServer(id: XoServer['id']): Promise<void>
|
||||
getAllGroups(): Promise<XoGroup[]>
|
||||
getAllProxies(): Promise<XoProxy[]>
|
||||
getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
|
||||
getAllJobs(type?: string): Promise<AnyXoJob[]>
|
||||
getProxy(id: XoProxy['id']): Promise<XoProxy>
|
||||
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
|
||||
getAllRemotes(): Promise<XoBackupRepository[]>
|
||||
getAllRemotesInfo(): Promise<
|
||||
Record<
|
||||
XoBackupRepository['id'],
|
||||
{
|
||||
size?: number
|
||||
used?: number
|
||||
available?: number
|
||||
encryption?: {
|
||||
algorithm: string
|
||||
isLegacy: boolean
|
||||
recommanded: string
|
||||
}
|
||||
}
|
||||
>
|
||||
>
|
||||
getAllSchedules(): Promise<XoSchedule[]>
|
||||
getAllUsers(): Promise<XoUser[]>
|
||||
getAllXenServers(): Promise<XoServer[]>
|
||||
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
|
||||
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
|
||||
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
|
||||
getBackupNgLogsSorted(opts: {
|
||||
after?: number
|
||||
before?: number
|
||||
filter: (log: AnyXoLog) => boolean
|
||||
limit?: number
|
||||
}): Promise<AnyXoLog[]>
|
||||
getGroup(id: XoGroup['id']): Promise<XoGroup>
|
||||
getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
|
||||
getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
|
||||
getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
|
||||
getObjectsByType: <T extends XapiXoRecord>(
|
||||
type: T['type'],
|
||||
opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
|
||||
) => Record<T['id'], T> | undefined
|
||||
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
|
||||
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
|
||||
getUser: (id: XoUser['id']) => Promise<XoUser>
|
||||
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
|
||||
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
|
||||
getXapiObject: <T extends XapiXoRecord>(
|
||||
maybeId: T['id'] | T,
|
||||
type: T['type'] | T['type'][]
|
||||
) => XapiRecordByXapiXoRecord[T['type']]
|
||||
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
|
||||
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
|
||||
getXenServer(id: XoServer['id']): Promise<XoServer>
|
||||
hasFeatureAuthorization(featureCode: string): Promise<boolean>
|
||||
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
|
||||
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
|
||||
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
|
||||
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
|
||||
}>
|
||||
listVmBackupsNg(
|
||||
backupRepositoryIds: XoBackupRepository['id'][],
|
||||
opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
|
||||
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
|
||||
/** Allow to add a new server in the DB (XCP-ng/XenServer) */
|
||||
registerXenServer(
|
||||
body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
|
||||
allowUnauthorized?: XoServer['allowUnauthorized']
|
||||
password: string
|
||||
readOnly?: XoServer['readOnly']
|
||||
}
|
||||
): Promise<XoServer>
|
||||
rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise<void>
|
||||
rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise<void>
|
||||
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
|
||||
runJob(job: AnyXoJob, schedule: XoSchedule): void
|
||||
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
|
||||
/** Remove a server from the DB (XCP-ng/XenServer) */
|
||||
unregisterXenServer(id: XoServer['id']): Promise<void>
|
||||
updateUser(
|
||||
id: XoUser['id'],
|
||||
updates: {
|
||||
/**
|
||||
* @deprecated
|
||||
*/
|
||||
email?: string
|
||||
authProviders?: Record<string, string>
|
||||
name?: string
|
||||
password?: string
|
||||
permission?: string
|
||||
preferences?: Record<string, string>
|
||||
}
|
||||
): Promise<void>
|
||||
updateGroup(
|
||||
id: XoGroup['id'],
|
||||
updates: {
|
||||
name?: string
|
||||
}
|
||||
): void
|
||||
getAllXapis(): Record<string, XapiConnection>
|
||||
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
|
||||
}
|
||||
@@ -637,6 +637,21 @@ export type XoTask = {
|
||||
method?: string
|
||||
name?: string
|
||||
objectId?: string
|
||||
objectType?:
|
||||
| XapiXoRecord['type']
|
||||
| 'backup'
|
||||
| 'backup-archive'
|
||||
| 'backup-job'
|
||||
| 'backup-log'
|
||||
| 'backup-repository'
|
||||
| 'group'
|
||||
| 'proxy'
|
||||
| 'restore'
|
||||
| 'restore-log'
|
||||
| 'schedule'
|
||||
| 'server'
|
||||
| 'task'
|
||||
| 'user'
|
||||
params?: Record<string, unknown>
|
||||
progress?: number
|
||||
type?: string
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/backups": "^0.67.2",
|
||||
"@xen-orchestra/fs": "^4.6.5",
|
||||
"@xen-orchestra/backups": "^0.68.2",
|
||||
"@xen-orchestra/fs": "^4.6.6",
|
||||
"filenamify": "^6.0.0",
|
||||
"getopts": "^2.2.5",
|
||||
"lodash": "^4.17.15",
|
||||
@@ -28,7 +28,7 @@
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"version": "1.1.6",
|
||||
"version": "1.1.7",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
|
||||
@@ -18,7 +18,7 @@ import groupBy from 'lodash/groupBy.js'
|
||||
import pDefer from 'promise-toolbox/defer'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
import reduce from 'lodash/reduce.js'
|
||||
import tar from 'tar'
|
||||
import * as tar from 'tar'
|
||||
import zlib from 'zlib'
|
||||
|
||||
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
|
||||
@@ -31,7 +31,7 @@ import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from '
|
||||
import { lvs, pvs } from './_lvm.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
|
||||
import { RemoteVhd } from './disks/RemoteVhd.mjs'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { ReadAhead } from '@xen-orchestra/disk-transform'
|
||||
@@ -44,7 +44,7 @@ const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
|
||||
|
||||
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
|
||||
|
||||
const compareTimestamp = (a, b) => a.timestamp - b.timestamp
|
||||
export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -613,6 +613,11 @@ export class RemoteAdapter {
|
||||
// if cache is missing or broken => regenerate it and return
|
||||
|
||||
async _readCacheListVmBackups(vmUuid) {
|
||||
// immutable remote can't use any caching
|
||||
// since the cache file may be non modifiable
|
||||
if (this._handler.isImmutable()) {
|
||||
return this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
|
||||
}
|
||||
const path = this.#getVmBackupsCache(vmUuid)
|
||||
|
||||
const cache = await this._readCache(path)
|
||||
@@ -739,7 +744,7 @@ export class RemoteAdapter {
|
||||
if (useChain) {
|
||||
disk = await openDiskChain({ handler, path })
|
||||
} else {
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
}
|
||||
disk = new ReadAhead(disk)
|
||||
|
||||
@@ -235,7 +235,7 @@ test('it merges delta of non destroyed chain', async () => {
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children after the merge
|
||||
assert.equal(metadata.size, 104960)
|
||||
assert.equal(metadata.size, 104448)
|
||||
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
// only check deletion
|
||||
@@ -402,13 +402,12 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
|
||||
assert.deepEqual(metadata.size, 4404224)
|
||||
|
||||
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
// grand child and clean vhd should not have changed
|
||||
const survivors = await handler.list(basePath)
|
||||
// console.log(survivors)
|
||||
if (useAlias) {
|
||||
const dataSurvivors = await handler.list(basePath + '/data')
|
||||
// the goal of the alias : do not move a full folder
|
||||
|
||||
@@ -6,7 +6,9 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
import { basename, dirname, resolve } from 'node:path'
|
||||
import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { mergeVhdChain } from 'vhd-lib/merge.js'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
|
||||
import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
|
||||
|
||||
import { Task } from './Task.mjs'
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
@@ -51,7 +53,15 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
|
||||
}
|
||||
}, 10e3)
|
||||
try {
|
||||
return await mergeVhdChain(handler, chain, {
|
||||
const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
|
||||
|
||||
const childDisks = []
|
||||
for (const path of chain) {
|
||||
childDisks.push(new RemoteVhdDisk({ handler, path }))
|
||||
}
|
||||
const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
|
||||
logInfo,
|
||||
mergeBlockConcurrency,
|
||||
onProgress({ done: d, total: t }) {
|
||||
@@ -60,6 +70,14 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
|
||||
},
|
||||
removeUnused: remove,
|
||||
})
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
|
||||
await parentDisk.init({ force: isResumingMerge })
|
||||
await childDiskChain.init({ force: isResumingMerge })
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
|
||||
|
||||
return result
|
||||
} finally {
|
||||
clearInterval(handle)
|
||||
}
|
||||
@@ -521,14 +539,14 @@ export async function cleanVm(
|
||||
const metadataWithMergedVhd = {}
|
||||
const doMerge = async () => {
|
||||
await asyncMap(toMerge, async chain => {
|
||||
const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
|
||||
const { finalDiskSize } = await limitedMergeVhdChain(handler, chain, {
|
||||
logInfo,
|
||||
logWarn,
|
||||
remove,
|
||||
mergeBlockConcurrency,
|
||||
})
|
||||
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,58 @@
|
||||
// @ts-check
|
||||
|
||||
import moment from 'moment-timezone'
|
||||
import assert from 'node:assert'
|
||||
/**
|
||||
* @typedef {import('moment-timezone').Moment} Moment
|
||||
*/
|
||||
/**
|
||||
* @typedef {Object} LTRSettings
|
||||
* @property {number} [firstHourOfTheDay] - First hour of the day (for daily retention)
|
||||
* @property {number} [firstDayOfWeek] - First day of the week (for weekly retention)
|
||||
* @property {number} [firstDayOfMonth] - First day of the month (for monthly retention)
|
||||
* @property {number} [firstDayOfYear] - First day of the year (for yearly retention)
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LTRConfig
|
||||
* @property {number} retention - Number of time buckets to retain
|
||||
* @property {LTRSettings} [settings] - Optional settings for the retention period
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LongTermRetention
|
||||
* @property {LTRConfig} [daily] - Daily retention configuration
|
||||
* @property {LTRConfig} [weekly] - Weekly retention configuration
|
||||
* @property {LTRConfig} [monthly] - Monthly retention configuration
|
||||
* @property {LTRConfig} [yearly] - Yearly retention configuration
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} Entry
|
||||
* @property {number | undefined} timestamp - Unix timestamp of the entry
|
||||
* @property {string} id
|
||||
* @property {*} [key] - Any other properties
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DateBucket
|
||||
* @property {number} remaining - Number of remaining buckets to fill
|
||||
* @property {string|null} lastMatchingBucket - Last bucket key that was matched
|
||||
* @property {(date: Moment) => string} formatter - Function to format date into bucket key
|
||||
* @property {Object.<string, Entry>} entries - Map of bucket keys to entries
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LTRDefinition
|
||||
* @property {(options: {firstHourOfTheDay?: number, firstDayOfWeek?: number, firstDayOfMonth?: number, firstDayOfYear?: number, dateCreator: (date: Moment) => Moment}) => (date: Moment) => string} makeDateFormatter
|
||||
* @property {string} [ancestor] - Name of the parent retention period
|
||||
*/
|
||||
|
||||
/**
|
||||
* Creates a function that converts dates to moment objects in a specific timezone
|
||||
* @param {string|undefined} timezone - IANA timezone string (e.g., 'America/New_York')
|
||||
* @returns {(date: Moment | number) => Moment} Function that creates timezoned moment objects
|
||||
*/
|
||||
|
||||
function instantiateTimezonedDateCreator(timezone) {
|
||||
return date => {
|
||||
@@ -9,9 +62,12 @@ function instantiateTimezonedDateCreator(timezone) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {Object.<string, LTRDefinition>}
|
||||
*/
|
||||
const LTR_DEFINITIONS = {
|
||||
daily: {
|
||||
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.hour(copy.hour() - firstHourOfTheDay)
|
||||
@@ -20,11 +76,11 @@ const LTR_DEFINITIONS = {
|
||||
},
|
||||
},
|
||||
weekly: {
|
||||
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
|
||||
copy.date(date.date() - firstDayOfWeek)
|
||||
copy.date(copy.date() - firstDayOfWeek)
|
||||
// warning, the year in term of week may different from YYYY
|
||||
// since the computation of the first week of a year is timezone dependent
|
||||
return copy.format('gggg-ww')
|
||||
@@ -33,7 +89,7 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'daily',
|
||||
},
|
||||
monthly: {
|
||||
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.date(copy.date() - firstDayOfMonth)
|
||||
@@ -43,7 +99,7 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'weekly',
|
||||
},
|
||||
yearly: {
|
||||
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.date(copy.date() - firstDayOfYear)
|
||||
@@ -53,27 +109,23 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'monthly',
|
||||
},
|
||||
}
|
||||
|
||||
// returns all entries but the last retention-th
|
||||
/**
|
||||
* return the entries too old to be kept
|
||||
* if multiple entries are i the same time bucket : keep only the most recent one
|
||||
* if an entry is valid in any of the bucket OR the minRetentionCount : keep it
|
||||
* if a bucket is completely empty : it does not count as one, thus it may extend the retention
|
||||
* @returns Array<Backup>
|
||||
* Groups entries into date buckets based on long-term retention configuration
|
||||
* @template {Entry} T
|
||||
* @param {Array<T>} entries - Array of entries sorted in descending order by timestamp
|
||||
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
|
||||
* @param {string | undefined} timezone - IANA timezone string
|
||||
* @returns {Object.<string, DateBucket>} Map of retention period names to their date buckets
|
||||
*/
|
||||
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
|
||||
assert.strictEqual(
|
||||
typeof minRetentionCount,
|
||||
'number',
|
||||
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
assert.strictEqual(
|
||||
minRetentionCount >= 0,
|
||||
true,
|
||||
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
|
||||
export function getLtrEntries(entries, longTermRetention, timezone) {
|
||||
/**
|
||||
* @type {Object.<string, DateBucket>}
|
||||
*/
|
||||
const dateBuckets = {}
|
||||
if (!longTermRetention || Object.keys(longTermRetention).length === 0) {
|
||||
return {}
|
||||
}
|
||||
const dateCreator = instantiateTimezonedDateCreator(timezone)
|
||||
// only check buckets that have a retention set
|
||||
for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
|
||||
@@ -81,7 +133,7 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
|
||||
assert.notStrictEqual(
|
||||
timezone,
|
||||
undefined,
|
||||
`timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
|
||||
`timezone must defined for ltr, got ${JSON.stringify({ longTermRetention, timezone })}`
|
||||
)
|
||||
assert.strictEqual(
|
||||
typeof retention,
|
||||
@@ -101,45 +153,92 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
|
||||
}
|
||||
}
|
||||
const nb = entries.length
|
||||
const minDurationEntries = []
|
||||
let previousTimestamp = -1
|
||||
for (let i = nb - 1; i >= 0; i--) {
|
||||
const entry = entries[i]
|
||||
if (entry.timestamp !== undefined) {
|
||||
// we go through the entries from the last (most recent) to the first (oldest)
|
||||
assert.ok(
|
||||
previousTimestamp === -1 || entry.timestamp < previousTimestamp,
|
||||
`entries must be sorted in desc order ${new Date(entry.timestamp)} , previous : > ${new Date(previousTimestamp)} `
|
||||
)
|
||||
previousTimestamp = entry.timestamp
|
||||
const entryDate = dateCreator(entry.timestamp)
|
||||
for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
|
||||
const bucket = formatter(entryDate)
|
||||
if (lastMatchingBucket !== bucket) {
|
||||
if (remaining === 0) {
|
||||
continue
|
||||
}
|
||||
dateBuckets[duration].lastMatchingBucket = bucket
|
||||
dateBuckets[duration].remaining -= 1
|
||||
}
|
||||
dateBuckets[duration].entries[bucket] = entry
|
||||
}
|
||||
} else {
|
||||
// replicated VM entries or snapshot retention don't have a timestamp
|
||||
// but also can't have LTR
|
||||
assert.deepStrictEqual(
|
||||
longTermRetention,
|
||||
{},
|
||||
"Can't compute long term retention if entries don't have a timestamp"
|
||||
// force it so the compiler is sure we have a timestamp
|
||||
if (entry.timestamp === undefined) {
|
||||
assert.notStrictEqual(
|
||||
entry.timestamp,
|
||||
undefined,
|
||||
`Can't compute long term retention if entries don't have a timestamp`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// also keep it on retention
|
||||
if (i >= nb - minRetentionCount) {
|
||||
minDurationEntries.push(entry)
|
||||
// we go through the entries from the last (most recent) to the first (oldest)
|
||||
assert.ok(
|
||||
previousTimestamp === -1 || entry.timestamp < previousTimestamp,
|
||||
`entries must be sorted in desc order ${new Date(entry.timestamp)} , previous : > ${new Date(previousTimestamp)} `
|
||||
)
|
||||
previousTimestamp = entry.timestamp
|
||||
const entryDate = dateCreator(entry.timestamp)
|
||||
for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
|
||||
const bucket = formatter(entryDate)
|
||||
if (lastMatchingBucket !== bucket) {
|
||||
if (remaining === 0) {
|
||||
continue
|
||||
}
|
||||
dateBuckets[duration].lastMatchingBucket = bucket
|
||||
dateBuckets[duration].remaining -= 1
|
||||
}
|
||||
dateBuckets[duration].entries[bucket] = entry
|
||||
}
|
||||
}
|
||||
const kept = new Set(minDurationEntries)
|
||||
|
||||
return dateBuckets
|
||||
}
|
||||
|
||||
/**
|
||||
* return the status of an entry
|
||||
* @param {Array<Entry>} entries
|
||||
* @param {Entry} entry
|
||||
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
|
||||
* @param {string | undefined} timezone - IANA timezone string
|
||||
* @returns {Array<{duration: string, dateBucket:string}>}
|
||||
*/
|
||||
export function getEntryStatus(entries, entry, longTermRetention, timezone) {
|
||||
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
|
||||
const entryBuckets = []
|
||||
for (const [duration, { entries }] of Object.entries(dateBuckets)) {
|
||||
for (const [dateBucket, bucketEntry] of Object.entries(entries)) {
|
||||
if (entry.id === bucketEntry.id) {
|
||||
entryBuckets.push({ duration, dateBucket })
|
||||
}
|
||||
}
|
||||
}
|
||||
return entryBuckets
|
||||
}
|
||||
|
||||
/**
|
||||
* return the entries too old to be kept
|
||||
* if multiple entries are in the same time bucket: keep only the oldest one
|
||||
* if an entry is valid in any of the bucket OR the minRetentionCount: keep it
|
||||
* if a bucket is completely empty: it does not count as one, thus it may extend the retention
|
||||
* if an entry is within the most recents minRetentionCount entries : keep it
|
||||
*
|
||||
* @param {number} minRetentionCount - Minimum number of most recent entries to always keep
|
||||
* @param {Array<Entry>} entries - Array of entries sorted in descending order by timestamp
|
||||
* @param {Object} options - Additional options
|
||||
* @param {LongTermRetention} [options.longTermRetention={}] - Configuration for retention periods
|
||||
* @param {string | undefined} [options.timezone] - IANA timezone string
|
||||
* @returns {Array<Entry>} Array of entries that can be removed
|
||||
*/
|
||||
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
|
||||
assert.strictEqual(
|
||||
typeof minRetentionCount,
|
||||
'number',
|
||||
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
assert.ok(
|
||||
minRetentionCount >= 0,
|
||||
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
|
||||
|
||||
const kept = new Set(entries.filter((_, index) => index >= entries.length - minRetentionCount))
|
||||
/**
|
||||
* @type {Set<Entry>}
|
||||
*/
|
||||
for (const { entries } of Object.values(dateBuckets)) {
|
||||
for (const entry of Object.values(entries)) {
|
||||
kept.add(entry)
|
||||
@@ -147,7 +246,7 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
|
||||
}
|
||||
|
||||
// ensure order is the same as the source
|
||||
const oldEntries = entries.filter(entry => {
|
||||
const oldEntries = entries.filter((entry, index) => {
|
||||
return !kept.has(entry)
|
||||
})
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ describe('_getOldEntries() should succeed', () => {
|
||||
expectedIds: [1],
|
||||
testLabel: 'should handle number based retention without timestamp ',
|
||||
},
|
||||
|
||||
{
|
||||
args: [
|
||||
0,
|
||||
|
||||
@@ -291,10 +291,16 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
await xapi.VTPM_create({ VM: vmRef, contents })
|
||||
})
|
||||
)
|
||||
|
||||
const vm = await xapi.getRecord('VM', vmRef)
|
||||
await Promise.all([
|
||||
incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
|
||||
xapi.setField('VM', vmRef, 'name_label', incrementalVm.vm.name_label),
|
||||
vmRecord.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
|
||||
xapi.setField('VM', vmRef, 'name_label', vmRecord.name_label),
|
||||
// correctly unlock the VM and reapply the target blocked operations
|
||||
vm.update_blocked_operations({
|
||||
start: null,
|
||||
start_on: null,
|
||||
...vmRecord.blocked_operations,
|
||||
}),
|
||||
])
|
||||
|
||||
return vmRef
|
||||
|
||||
@@ -91,39 +91,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* used to ensure compatibility with the previous snapshots that were having the config stored only into VM
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export async function populateVdisOtherConfig(xapi, vmRef) {
|
||||
const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
|
||||
const {
|
||||
[DATETIME]: datetime,
|
||||
[DELTA_CHAIN_LENGTH]: chainLength,
|
||||
[EXPORTED_SUCCESSFULLY]: successfully,
|
||||
[JOB_ID]: jobId,
|
||||
[REPLICATED_TO_SR_UUID]: replicatedTo,
|
||||
[SCHEDULE_ID]: scheduleId,
|
||||
[VM_UUID]: vmUuid,
|
||||
} = otherConfig
|
||||
|
||||
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
|
||||
xapi.setFieldEntries(type, ref, 'other_config', {
|
||||
[DATETIME]: datetime,
|
||||
[DELTA_CHAIN_LENGTH]: chainLength,
|
||||
[EXPORTED_SUCCESSFULLY]: successfully,
|
||||
[JOB_ID]: jobId,
|
||||
[REPLICATED_TO_SR_UUID]: replicatedTo,
|
||||
[SCHEDULE_ID]: scheduleId,
|
||||
[VM_UUID]: vmUuid,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* set the other_config key related to a backup of a VM and its associated VDIs
|
||||
|
||||
@@ -2,10 +2,11 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
|
||||
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
||||
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
|
||||
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
||||
|
||||
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
|
||||
_getRemoteWriter() {
|
||||
return FullRemoteWriter
|
||||
_getRemoteWriters() {
|
||||
return [FullRemoteWriter, AggregatedFullRemoteWriter]
|
||||
}
|
||||
|
||||
_filterTransferList(transferList) {
|
||||
|
||||
@@ -5,12 +5,14 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
||||
import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
|
||||
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import { AbstractXapi } from './_AbstractXapi.mjs'
|
||||
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
||||
import { AggregatedFullXapiWriter } from '../_writers/AggregatedFullXapiWriter.mjs'
|
||||
|
||||
const { debug } = createLogger('xo:backups:FullXapiVmBackup')
|
||||
|
||||
export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
_getWriters() {
|
||||
return [FullRemoteWriter, FullXapiWriter]
|
||||
return [FullRemoteWriter, FullXapiWriter, AggregatedFullRemoteWriter, AggregatedFullXapiWriter]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
@@ -64,9 +66,16 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
const vdi = await this._xapi.getRecord('VDI', vdiRef)
|
||||
|
||||
// the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
|
||||
// of the real stream size in general, since a disk is never completely full
|
||||
// of the real stream size in general, since a disk is rarely completely full
|
||||
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
|
||||
maxStreamLength += vdi.virtual_size
|
||||
|
||||
// xva files are tar archive with data cut in 1MB blocks
|
||||
// each block is signed with a hash (in xxhash format)
|
||||
// each of these blocks + hash add one tar header entry in the file
|
||||
const XVA_BLOCK_SIZE = 1024 * 1024
|
||||
const nbBlocks = Math.ceil(vdi.virtual_size / XVA_BLOCK_SIZE)
|
||||
const headersSize = nbBlocks * (512 /* file header */ + 512 /* xxhash header */ + 512) /* xhxhash size */
|
||||
maxStreamLength += XVA_BLOCK_SIZE * nbBlocks + headersSize
|
||||
}
|
||||
|
||||
const sizeContainer = watchStreamSize(stream)
|
||||
|
||||
@@ -11,11 +11,12 @@ import { Disposable } from 'promise-toolbox'
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
|
||||
const { warn } = createLogger('xo:backups:Incrementalremote')
|
||||
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
_getRemoteWriter() {
|
||||
return IncrementalRemoteWriter
|
||||
_getRemoteWriters() {
|
||||
return [IncrementalRemoteWriter, AggregatedIncrementalRemoteWriter]
|
||||
}
|
||||
|
||||
// we'll transfer the full list if at least one backup should be transferred
|
||||
|
||||
@@ -15,12 +15,19 @@ import {
|
||||
markExportSuccessfull,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
|
||||
|
||||
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
|
||||
_getWriters() {
|
||||
return [IncrementalRemoteWriter, IncrementalXapiWriter]
|
||||
return [
|
||||
IncrementalRemoteWriter,
|
||||
IncrementalXapiWriter,
|
||||
AggregatedIncrementalRemoteWriter,
|
||||
AggregatedIncrementalXapiWriter,
|
||||
]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
|
||||
@@ -45,26 +45,41 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
const writers = new Set()
|
||||
this._writers = writers
|
||||
|
||||
const RemoteWriter = this._getRemoteWriter()
|
||||
extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
|
||||
const adapter = remoteAdapters[remoteId]
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
|
||||
if (settings.distributeBackups) {
|
||||
writers.add(
|
||||
new RemoteWriter({
|
||||
adapter,
|
||||
new AggregratedBackupWriter({
|
||||
adapters: remoteAdapters,
|
||||
BackupWriter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
})
|
||||
} else {
|
||||
extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
|
||||
const adapter = remoteAdapters[remoteId]
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
const { filter } = job
|
||||
if (filter === undefined) {
|
||||
this._filterPredicate = () => true
|
||||
|
||||
@@ -3,22 +3,18 @@ import groupBy from 'lodash/groupBy.js'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { defer } from 'golike-defer'
|
||||
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import { Abstract } from './_Abstract.mjs'
|
||||
import {
|
||||
DATETIME,
|
||||
JOB_ID,
|
||||
SCHEDULE_ID,
|
||||
populateVdisOtherConfig,
|
||||
resetVmOtherConfig,
|
||||
setVmOtherConfig,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { DATETIME, JOB_ID, SCHEDULE_ID, VM_UUID, resetVmOtherConfig, setVmOtherConfig } from '../../_otherConfig.mjs'
|
||||
|
||||
const { warn } = createLogger('xo:backups:AbstractXapi')
|
||||
const { warn, info } = createLogger('xo:backups:AbstractXapi')
|
||||
|
||||
const TEMP_SNAPSHOT_NAME = 'xo-backup-temp-snapshot-name'
|
||||
|
||||
export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
constructor({
|
||||
@@ -86,48 +82,84 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
const writers = new Set()
|
||||
this._writers = writers
|
||||
|
||||
const [BackupWriter, ReplicationWriter] = this._getWriters()
|
||||
const [BackupWriter, ReplicationWriter, AggregratedBackupWriter, AggregratedReplicationWriter] =
|
||||
this._getWriters()
|
||||
|
||||
const allSettings = job.settings
|
||||
Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
|
||||
if (settings.exportRetention > 0) {
|
||||
if (settings.distributeBackups) {
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
new AggregratedBackupWriter({
|
||||
adapters: remoteAdapters,
|
||||
BackupWriter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid: vm.uuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
} else {
|
||||
Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid: vm.uuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
srs.forEach(sr => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[sr.uuid],
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
}
|
||||
if (settings.copyRetention) {
|
||||
if (settings.distributeReplications) {
|
||||
writers.add(
|
||||
new ReplicationWriter({
|
||||
new AggregratedReplicationWriter({
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
ReplicationWriter,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid: vm.uuid,
|
||||
sr,
|
||||
settings: targetSettings,
|
||||
srs,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
} else {
|
||||
srs.forEach(sr => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[sr.uuid],
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
writers.add(
|
||||
new ReplicationWriter({
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
vmUuid: vm.uuid,
|
||||
sr,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,7 +195,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
|
||||
const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
|
||||
ignoredVdisTag: '[NOBAK]',
|
||||
name_label: this._getSnapshotNameLabel(vm),
|
||||
name_label: TEMP_SNAPSHOT_NAME,
|
||||
unplugVusbs: true,
|
||||
})
|
||||
this.timestamp = Date.now()
|
||||
@@ -173,6 +205,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
scheduleId: this.scheduleId,
|
||||
vmUuid: vm.uuid,
|
||||
})
|
||||
const snapshot = await xapi.getRecord('VM', snapshotRef)
|
||||
await snapshot.set_name_label(this._getSnapshotNameLabel(vm))
|
||||
// reload data to ensure it is up to date with the new name label
|
||||
this._exportedVm = await xapi.getRecord('VM', snapshotRef)
|
||||
return this._exportedVm.uuid
|
||||
})
|
||||
@@ -182,43 +217,113 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
}
|
||||
|
||||
// handle snapshot by VDI since snapshot of cbtDestroySnapshotData jobs
|
||||
// aren't attached to any VM
|
||||
|
||||
// look at all the vdi snapshots ( snapshot_of not empty )
|
||||
// with the same vm_uuid and job_uuid
|
||||
// for cbt_metadata list them unconditionnaly to remove older one
|
||||
// for other: only list them if they are attached to a VM snapshot
|
||||
// and if this vm snapshot is also part of the backup
|
||||
// ensure they are attached to only one vm snapshot
|
||||
// ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
|
||||
|
||||
async _fetchJobSnapshots() {
|
||||
const jobId = this._jobId
|
||||
const vmRef = this._vm.$ref
|
||||
const xapi = this._xapi
|
||||
|
||||
// to ensure compatibility with snapshots older than CBT implementation
|
||||
// update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
|
||||
// remove by then end of 2024
|
||||
const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
|
||||
const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
|
||||
const vdiCandidates = {}
|
||||
|
||||
const vmSnapshots = []
|
||||
vmSnapshotsOtherConfig.forEach((other_config, i) => {
|
||||
if (other_config[JOB_ID] === jobId) {
|
||||
vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
|
||||
}
|
||||
})
|
||||
await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
|
||||
// end of compatibility handling
|
||||
Object.values(xapi.objects.indexes.type.VDI)
|
||||
.filter(_ => !!_) // filter nullish
|
||||
.filter(({ other_config, $snapshot_of }) => {
|
||||
return $snapshot_of !== undefined && other_config[JOB_ID] === jobId && other_config[VM_UUID] === this._vm.uuid
|
||||
})
|
||||
.forEach(vdi => {
|
||||
vdiCandidates[vdi.uuid] = vdi
|
||||
})
|
||||
|
||||
// handle snapshot by VDI
|
||||
this._jobSnapshotVdis = []
|
||||
const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
|
||||
for (const srcVdi of srcVdis) {
|
||||
const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
|
||||
for (const snapshot of snapshots) {
|
||||
// only keep the snapshot related to this backup job
|
||||
// and only if the job is still using purge snapshot data or if the disk
|
||||
// is not a cbt metadata disk ( expect a type: user for normal disks)
|
||||
if (
|
||||
snapshot.other_config[JOB_ID] === jobId &&
|
||||
(this._settings.cbtDestroySnapshotData || snapshot.type !== 'cbt_metadata')
|
||||
) {
|
||||
this._jobSnapshotVdis.push(snapshot)
|
||||
}
|
||||
// check that user snapshots are clean
|
||||
|
||||
for (const vdi of Object.values(vdiCandidates)) {
|
||||
// cbt metadata are always considered linked to a backup job
|
||||
// if they have the right other_config
|
||||
if (vdi.type === 'cbt_metadata') {
|
||||
continue
|
||||
}
|
||||
const vbds = vdi.$VBDs
|
||||
.filter(({ $VM }) => !!$VM) // filter empty VMs
|
||||
.filter(({ $VM }) => $VM.is_control_domain === false) // do not handle control domain
|
||||
if (vbds.length === 0) {
|
||||
// orphan vdi snapshot
|
||||
info(
|
||||
`disk snapshot ${vdi.name_label} is orphan or attached only to control domain,
|
||||
it will be removed at the end of a successful backup run`,
|
||||
{ vdi, attachedto: vdi.$VBDs.map(vbd => vbd?.$VM) }
|
||||
)
|
||||
continue
|
||||
}
|
||||
const userVms = vbds.map(({ $VM }) => $VM)
|
||||
if (vbds.length > 1) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, userVms }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
const vm = vbds[0].$VM
|
||||
|
||||
// xapi has some issue regarding vdi snapshot attached to non snapshot VM
|
||||
// it can also be some user action forcibely linking a vdi snapshot to a vm
|
||||
// => we exclude these from the backup processing
|
||||
if (vm.$snapshot_of === undefined) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
// vdi is attached only to a snapshot that is not a backup snapshot
|
||||
// we don't check scheduleId since we are looking for all the snapshot of this job
|
||||
// => excludes from the list to be cleared
|
||||
if (
|
||||
vm.other_config[DATETIME] !== vdi.other_config[DATETIME] ||
|
||||
vm.other_config[JOB_ID] !== vdi.other_config[JOB_ID] ||
|
||||
vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
|
||||
) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
// check if all the disks of these VM snapshot have been harvested
|
||||
// if not => remove it from the list to ensure we won't half destroy VM later
|
||||
vm.$VBDs
|
||||
.filter(({ $VDI }) => !!$VDI) // filter missing keys
|
||||
.filter(({ $VDI }) => $VDI && vdiCandidates[$VDI.uuid] === undefined)
|
||||
.forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
|
||||
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
|
||||
is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm, vbds, outOfSnapshotsVdi }
|
||||
)
|
||||
// this will be called multiple time but it is not really an issue
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
})
|
||||
}
|
||||
|
||||
this._jobSnapshotVdis = Object.values(vdiCandidates)
|
||||
}
|
||||
|
||||
async _removeUnusedSnapshots() {
|
||||
@@ -268,7 +373,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// since we won't be able to remove an attached VDI
|
||||
assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
|
||||
const vdiVm = vbds[0].$VM
|
||||
if (!vdiVm.is_a_snapshot) {
|
||||
if (vdiVm.$snapshot_of === undefined) {
|
||||
// don't delete a VM (especially a control domain)
|
||||
warn(
|
||||
`VM ${vdiVm.uuid} (${vdiVm.name_label}) linked to VDI ${vdi.uuid} (${vdi.name_label}) should be a snapshot`
|
||||
@@ -308,6 +413,11 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// list and remove the snapshot were the jobs failed between
|
||||
// makesnapshot and update_other_config
|
||||
const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)
|
||||
await asyncEach(snapshots, snapshot => snapshot.$destroy())
|
||||
}
|
||||
|
||||
async _removeSnapshotData() {
|
||||
@@ -315,8 +425,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// going back to a previous version of XO not supporting CBT will create a full backup
|
||||
// this will only do something after snapshot and transfer
|
||||
if (
|
||||
this._exportedVm !== undefined &&
|
||||
// don't modify the VM
|
||||
this._exportedVm?.is_a_snapshot &&
|
||||
this._exportedVm.$snapshot_of !== undefined &&
|
||||
// user don't want to keep the snapshot data
|
||||
this._settings.snapshotRetention === 0 &&
|
||||
// preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
|
||||
|
||||
export class AggregatedFullRemoteWriter extends AbstractAggregatedRemoteWriter {
|
||||
async deleteOldBackupsOnAdapter(adapter, backups) {
|
||||
await adapter.deleteFullVmBackups(backups)
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
|
||||
await this.setOldBackups()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
async run(props) {
|
||||
const mainAdapter = await this.computeAdapterCandidate()
|
||||
this.mainWriter = this.getWriterByAdapter(mainAdapter)
|
||||
return this.mainWriter.run(props)
|
||||
}
|
||||
|
||||
// cleanup if deleteFirst is NOT selected
|
||||
// release the lock on every adapters with this VM
|
||||
async afterBackup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
|
||||
|
||||
export class AggregatedFullXapiWriter extends AbstractAggregatedXapiWriter {
|
||||
// delete replica if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
this.setOldReplicaList()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
}
|
||||
|
||||
async run(props) {
|
||||
const mainSr = await this.computeSRCandidate()
|
||||
this.mainWriter = this.getWriterBySr(mainSr)
|
||||
await this.mainWriter.run(props)
|
||||
}
|
||||
|
||||
// delete replica if not using deleteFirst
|
||||
async afterBackup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
|
||||
export class AggregatedIncrementalRemoteWriter extends AbstractAggregatedRemoteWriter {
|
||||
async deleteOldBackupsOnAdapter(adapter, backups) {
|
||||
await adapter.deleteDeltaVmBackups(backups)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Map<string,string>} baseUuidToSrcVdi
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
let selectedCopy = new Map()
|
||||
for (const writer of this.writers) {
|
||||
const copy = new Map(baseUuidToSrcVdi)
|
||||
await writer.checkBaseVdis(copy)
|
||||
if (copy.size > 0) {
|
||||
// there can be multiple candidates
|
||||
// we use the last one in this case
|
||||
this.mainWriter = writer
|
||||
selectedCopy = copy
|
||||
}
|
||||
}
|
||||
// update the parameter with our best candidate
|
||||
// that can be an empty map
|
||||
baseUuidToSrcVdi.forEach(key => {
|
||||
if (!selectedCopy.has(key)) {
|
||||
baseUuidToSrcVdi.delete(key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
|
||||
await this.setOldBackups()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
async prepare({ isFull }) {
|
||||
if (this.mainWriter === undefined) {
|
||||
// use the remote with the most empty space for starting a new chain
|
||||
const mainAdapter = await this.computeAdapterCandidate()
|
||||
this.mainWriter = this.getWriterByAdapter(mainAdapter)
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.prepare({ isFull })))
|
||||
}
|
||||
// write only on the main writer
|
||||
transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
|
||||
return this.mainWriter.transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot })
|
||||
}
|
||||
|
||||
// chain only on the main writer
|
||||
updateUuidAndChain({ isVhdDifferencing, vdis }) {
|
||||
return this.mainWriter.updateUuidAndChain({ isVhdDifferencing, vdis })
|
||||
}
|
||||
|
||||
// remove the backups and remove the entries
|
||||
async cleanup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.cleanup()))
|
||||
}
|
||||
// cleanVM on all, since we may want to delete a chani
|
||||
// that is not on the main writer
|
||||
async afterBackup() {
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AggregatedIncrementalXapiWriter')
|
||||
export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWriter {
|
||||
/**
|
||||
*
|
||||
* @param {Map<string,string>} baseUuidToSrcVdi
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
debug('checkBaseVdis', { baseUuidToSrcVdi })
|
||||
let selectedCopy = new Map()
|
||||
for (const writer of this.writers) {
|
||||
const copy = new Map(baseUuidToSrcVdi)
|
||||
await writer.checkBaseVdis(copy)
|
||||
if (copy.size > 0) {
|
||||
debug('checkBaseVdis found a mainwriter candidate ', writer._sr.name_label)
|
||||
// there can be multiple candidates
|
||||
// we use the last one in this case
|
||||
this.mainWriter = writer
|
||||
selectedCopy = copy
|
||||
} else {
|
||||
debug(writer._sr.name_label, 'is not a delta candidate')
|
||||
}
|
||||
}
|
||||
// update the parameter with our best candidate
|
||||
// that can be an empty map
|
||||
baseUuidToSrcVdi.forEach(key => {
|
||||
if (!selectedCopy.has(key)) {
|
||||
baseUuidToSrcVdi.delete(key)
|
||||
}
|
||||
})
|
||||
debug('checkBaseVdis', { baseUuidToSrcVdi })
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
this.setOldReplicaList()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
}
|
||||
|
||||
async prepare(args) {
|
||||
if (this.mainWriter === undefined) {
|
||||
debug('no mainwriter found, fallback to a new SR')
|
||||
// use the SR with the most empty space for starting a new chain
|
||||
const mainSr = await this.computeSRCandidate()
|
||||
this.mainWriter = this.getWriterBySr(mainSr)
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.prepare(args)))
|
||||
}
|
||||
|
||||
// write only on the main writer
|
||||
transfer(args) {
|
||||
return this.mainWriter.transfer(args)
|
||||
}
|
||||
|
||||
// chain only on the main writer
|
||||
updateUuidAndChain(args) {
|
||||
return this.mainWriter.updateUuidAndChain(args)
|
||||
}
|
||||
|
||||
// remove the backups and remove the entries
|
||||
async cleanup() {
|
||||
debug('cleanup')
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.cleanup()))
|
||||
}
|
||||
|
||||
// cleanVM on all, since we may want to delete a chani
|
||||
// that is not on the main writer
|
||||
async afterBackup() {
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -59,8 +59,8 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
xva: './' + dataBasename,
|
||||
}
|
||||
|
||||
const { deleteFirst } = settings
|
||||
if (deleteFirst) {
|
||||
const { skipDeleteOldEntries, deleteFirst } = settings
|
||||
if (!skipDeleteOldEntries && deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
@@ -73,9 +73,10 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
return { size: sizeContainer.size }
|
||||
})
|
||||
metadata.size = sizeContainer.size
|
||||
metadata.tags = await this.getLongTermRetentionTags(metadata)
|
||||
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
|
||||
|
||||
if (!deleteFirst) {
|
||||
if (!skipDeleteOldEntries && !deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
|
||||
@@ -46,8 +46,8 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
const oldVms = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
|
||||
|
||||
const deleteOldBackups = () => asyncMapSettled(oldVms, vm => xapi.VM_destroy(vm.$ref))
|
||||
const { deleteFirst, _warmMigration } = settings
|
||||
if (deleteFirst) {
|
||||
const { deleteFirst, skipDeleteOldEntries, _warmMigration } = settings
|
||||
if (!skipDeleteOldEntries && deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
}),
|
||||
])
|
||||
|
||||
if (!deleteFirst) {
|
||||
if (!skipDeleteOldEntries && !deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,13 +114,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
oldEntries.length = maxMergedDeltasPerRun
|
||||
}
|
||||
|
||||
if (settings.deleteFirst) {
|
||||
if (settings.deleteFirst && !settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
if (!this._settings.deleteFirst) {
|
||||
if (!this._settings.deleteFirst && !this._settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
@@ -244,6 +244,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
|
||||
return { size }
|
||||
})
|
||||
metadataContent.tags = await this.getLongTermRetentionTags(metadataContent)
|
||||
metadataContent.size = size // @todo return exactly the size written by this writer
|
||||
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { importIncrementalVm } from '../../_incrementalVm.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
@@ -9,8 +8,18 @@ import { Task } from '../../Task.mjs'
|
||||
import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
|
||||
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
|
||||
import { listReplicatedVms } from './_listReplicatedVms.mjs'
|
||||
import { COPY_OF, setVmOtherConfig, BASE_DELTA_VDI } from '../../_otherConfig.mjs'
|
||||
import {
|
||||
COPY_OF,
|
||||
setVmOtherConfig,
|
||||
BASE_DELTA_VDI,
|
||||
JOB_ID,
|
||||
SCHEDULE_ID,
|
||||
REPLICATED_TO_SR_UUID,
|
||||
DATETIME,
|
||||
VM_UUID,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import assert from 'node:assert'
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
|
||||
export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
@@ -74,7 +83,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
|
||||
this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
|
||||
|
||||
if (settings.deleteFirst) {
|
||||
if (settings.deleteFirst && settings.skipDeleteOldEntries) {
|
||||
// we want to keep the baseVM when copying a delta
|
||||
// even if we want to keep only one after
|
||||
let mostRecentEntry
|
||||
@@ -87,19 +96,37 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
await this._deleteOldEntries()
|
||||
if (!this._settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
|
||||
async _deleteOldEntries() {
|
||||
return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
|
||||
return asyncMapSettled(this._oldEntries, vm => vm.$destroy({ bypassBlockedOperation: true }))
|
||||
}
|
||||
|
||||
#decorateVmMetadata(backup) {
|
||||
#decorateVmMetadata(backup, timestamp) {
|
||||
const { _warmMigration } = this._settings
|
||||
const sr = this._sr
|
||||
const vm = backup.vm
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
|
||||
vm.name_label = `${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`
|
||||
// update other_config data as soon as possible to ensure the next job
|
||||
// will be able to detect any partial transfer and lean them
|
||||
vm.other_config[COPY_OF] = vm.uuid
|
||||
vm.other_config[JOB_ID] = job.id
|
||||
vm.other_config[SCHEDULE_ID] = scheduleId
|
||||
vm.other_config[REPLICATED_TO_SR_UUID] = sr.uuid
|
||||
// set the timestamp in the past to ensure any incomplete VM will be deleted on next run
|
||||
vm.other_config[DATETIME] = formatFilenameDate(0)
|
||||
|
||||
vm.blocked_operations = {
|
||||
start: 'Start operation for this vm is blocked, clone it if you want to use it.',
|
||||
start_on: 'Start operation for this vm is blocked, clone it if you want to use it.',
|
||||
}
|
||||
|
||||
if (!_warmMigration) {
|
||||
vm.tags.push('Continuous Replication')
|
||||
}
|
||||
@@ -118,6 +145,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
|
||||
Object.values(backup.vdis).forEach(vdi => {
|
||||
vdi.other_config[COPY_OF] = vdi.uuid
|
||||
vdi.other_config[JOB_ID] = job.id
|
||||
vdi.other_config[SCHEDULE_ID] = scheduleId
|
||||
vdi.other_config[REPLICATED_TO_SR_UUID] = sr.uuid
|
||||
vdi.other_config[VM_UUID] = vm.uuid
|
||||
|
||||
if (sourceVdiUuids.length > 0) {
|
||||
const baseReplicatedTo = replicatedVdis.filter(
|
||||
replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
|
||||
@@ -144,12 +176,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
const sr = this._sr
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
|
||||
const { uuid: srUuid, $xapi: xapi } = sr
|
||||
|
||||
let targetVmRef
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
|
||||
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport, timestamp), sr)
|
||||
// size is mandatory to ensure the task have the right data
|
||||
return {
|
||||
size: Object.values(deltaExport.disks).reduce(
|
||||
@@ -166,15 +197,8 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
!_warmMigration &&
|
||||
targetVm.ha_restart_priority !== '' &&
|
||||
Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
|
||||
targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
|
||||
asyncMap(['start', 'start_on'], op =>
|
||||
targetVm.update_blocked_operations(
|
||||
op,
|
||||
'Start operation for this vm is blocked, clone it if you want to use it.'
|
||||
)
|
||||
),
|
||||
setVmOtherConfig(xapi, targetVmRef, {
|
||||
timestamp,
|
||||
timestamp, // updated at the end to mark the transfer as complete
|
||||
jobId: job.id,
|
||||
scheduleId,
|
||||
vmUuid: vm.uuid,
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { compareTimestamp } from '../../RemoteAdapter.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AbstractAggregatedRemoteWriter')
|
||||
|
||||
export const DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY = 2
|
||||
export class AbstractAggregatedRemoteWriter {
|
||||
#BackupWriter
|
||||
#adapters
|
||||
get adapters() {
|
||||
return this.#adapters
|
||||
}
|
||||
|
||||
mainWriter
|
||||
#props
|
||||
get props() {
|
||||
return this.#props
|
||||
}
|
||||
#oldBackups
|
||||
|
||||
#writers = []
|
||||
get writers() {
|
||||
return this.#writers
|
||||
}
|
||||
constructor({ adapters, BackupWriter, ...props }) {
|
||||
debug('instantiate AggregatedFullRemoteWriter')
|
||||
if (Object.keys(adapters).length < 1) {
|
||||
throw new Error('AggregatedRemoteWriter need at least one backup repository to work ')
|
||||
}
|
||||
this.#props = props
|
||||
this.#adapters = adapters
|
||||
this.#BackupWriter = BackupWriter
|
||||
}
|
||||
|
||||
getWriterByAdapter(adapter) {
|
||||
for (const writer of this.#writers) {
|
||||
if (writer._adapter === adapter) {
|
||||
return writer
|
||||
}
|
||||
}
|
||||
throw new Error("Can't find writer for adapter")
|
||||
}
|
||||
|
||||
/**
|
||||
* select the best adapter candidate to write this backup
|
||||
* if all the adapter infos expose an available info use the one with the more free space
|
||||
* if none of the adapter exposes this info : choose at random
|
||||
* if mixed : throw an error
|
||||
* @returns {RemoteAdapter}
|
||||
*/
|
||||
async computeAdapterCandidate() {
|
||||
const adapters = Object.values(this.#adapters)
|
||||
const infos = await Promise.all(
|
||||
adapters.map(async adapter => {
|
||||
const infos = await adapter._handler.getInfo()
|
||||
return { adapter, infos }
|
||||
})
|
||||
)
|
||||
const withLimit = infos.filter(({ infos: { available } }) => available > 0).length
|
||||
if (withLimit !== 0 && withLimit !== infos.length) {
|
||||
throw new Error(`an Aggregate remote can't use a mix of limited and unlimited storages`)
|
||||
}
|
||||
let candidates
|
||||
if (withLimit === 0) {
|
||||
debug('no storage limit , choose at random')
|
||||
candidates = Object.values(this.adapters)
|
||||
} else {
|
||||
debug('storage limit , choose at random between those with the max space avilable')
|
||||
let maxSpace = 0
|
||||
infos.forEach(({ infos: { available } }) => {
|
||||
if (available > maxSpace) {
|
||||
maxSpace = available
|
||||
}
|
||||
})
|
||||
|
||||
// candidates are the adapter with the same free space , equal to the max free space
|
||||
candidates = infos.filter(({ infos }) => infos.available === maxSpace).map(({ adapter }) => adapter)
|
||||
}
|
||||
const adapterCandidate = candidates[Math.floor(Math.random() * candidates.length)]
|
||||
debug('adapter candidate is selected ', adapterCandidate._handler._remote.id)
|
||||
return adapterCandidate
|
||||
}
|
||||
|
||||
async setupWriters() {
|
||||
debug('Setup writers ')
|
||||
const settings = this.#props.settings
|
||||
const allSettings = this.#props.job.settings
|
||||
|
||||
Object.entries(this.#adapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
skipDeleteOldEntries: true, // delete is handled globally
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
const writer = new this.#BackupWriter({
|
||||
...this.#props,
|
||||
adapter,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
this.#writers.push(writer)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async getEntriesPerAdapter(adapter) {
|
||||
const scheduleId = this.#props.scheduleId
|
||||
const vmUuid = this.#props.vmUuid
|
||||
return (await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)).map(entry => ({
|
||||
...entry,
|
||||
adapter,
|
||||
}))
|
||||
}
|
||||
|
||||
async setOldBackups() {
|
||||
const entriesPerAdapter = await Promise.all(
|
||||
Object.values(this.#adapters).map(adapter => this.getEntriesPerAdapter(adapter))
|
||||
)
|
||||
const entries = entriesPerAdapter
|
||||
.flat(1)
|
||||
.filter(_ => !!_)
|
||||
.sort(compareTimestamp)
|
||||
|
||||
const settings = this.#props.settings
|
||||
this.#oldBackups = getOldEntries(settings.exportRetention - 1, entries, {
|
||||
longTermRetention: settings.longTermRetention,
|
||||
timezone: settings.timezone,
|
||||
})
|
||||
debug(`got ${this.#oldBackups.length} old backup(s) to delete `)
|
||||
}
|
||||
|
||||
async deleteOldBackupsOnAdapter(adapter, backup) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
async deleteOldBackups() {
|
||||
const byAdapters = new Map()
|
||||
this.#oldBackups.forEach(({ adapter, ...backup }) => {
|
||||
const current = byAdapters.has(adapter) ? byAdapters.get(adapter) : []
|
||||
current.push(backup)
|
||||
byAdapters.set(adapter, current)
|
||||
})
|
||||
await asyncEach(
|
||||
byAdapters.entries(),
|
||||
async ([adapter, backups]) => {
|
||||
debug(`delete ${backups.length} old backup(s) to delete from ${adapter._handler._remote.id}`)
|
||||
await this.deleteOldBackupsOnAdapter(adapter, backups)
|
||||
},
|
||||
{
|
||||
concurrency: this.#props.deleteConcurrency ?? DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { compareReplicatedVmDatetime, listReplicatedVms } from './_listReplicatedVms.mjs'
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AbstractAggregatedXapiWriter')
|
||||
export class AbstractAggregatedXapiWriter {
|
||||
#ReplicationWriter
|
||||
#storageRepositories
|
||||
|
||||
#writers = []
|
||||
get writers() {
|
||||
return this.#writers
|
||||
}
|
||||
mainWriter
|
||||
#props
|
||||
get props() {
|
||||
return this.#props
|
||||
}
|
||||
#oldVmReplicaList = []
|
||||
|
||||
constructor({ srs, ReplicationWriter, ...props }) {
|
||||
debug('instantiate AbstractAggregatedXapiWriter')
|
||||
if (srs.length < 1) {
|
||||
throw new Error('AggregatedXapiWriter need at least one storage repository to work ')
|
||||
}
|
||||
this.#props = props
|
||||
this.#ReplicationWriter = ReplicationWriter
|
||||
this.#storageRepositories = srs
|
||||
}
|
||||
|
||||
getWriterBySr(storageRepository) {
|
||||
for (const writer of this.#writers) {
|
||||
if (writer._sr === storageRepository) {
|
||||
return writer
|
||||
}
|
||||
}
|
||||
throw new Error("Can't find writer for storage repository")
|
||||
}
|
||||
|
||||
async computeSRCandidate() {
|
||||
let maxSpace = 0
|
||||
this.#storageRepositories.forEach(({ physical_utilisation, physical_size }) => {
|
||||
if (physical_size - physical_utilisation > maxSpace) {
|
||||
maxSpace = physical_size - physical_utilisation
|
||||
}
|
||||
})
|
||||
debug(`computeSRCandidate found the max free space ${maxSpace} for vm${this.#props.vmUuid}`)
|
||||
|
||||
const srCandidates = this.#storageRepositories.filter(
|
||||
({ physical_utilisation, physical_size }) => maxSpace === physical_size - physical_utilisation
|
||||
)
|
||||
// one at random from the sr with the most free space
|
||||
const candidate = srCandidates[Math.floor(Math.random() * srCandidates.length)]
|
||||
|
||||
debug(
|
||||
`computeSRCandidate found ${srCandidates.length} sr with ${maxSpace} free size, and chose ${candidate.name_label} for vm${this.#props.vmUuid}`
|
||||
)
|
||||
return candidate
|
||||
}
|
||||
|
||||
setOldReplicaList() {
|
||||
debug(`setOldReplicaList for vm ${this.#props.vmUuid}`)
|
||||
const scheduleId = this.#props.scheduleId
|
||||
const vmUuid = this.#props.vmUuid
|
||||
const settings = this.#props.settings
|
||||
|
||||
const replicatedVms = this.#storageRepositories
|
||||
.map(sr => {
|
||||
return listReplicatedVms(sr.$xapi, scheduleId, sr.uuid, vmUuid)
|
||||
})
|
||||
.flat(1)
|
||||
.filter(_ => !!_)
|
||||
.sort(compareReplicatedVmDatetime)
|
||||
|
||||
this.#oldVmReplicaList = getOldEntries(settings.copyRetention - 1, replicatedVms)
|
||||
|
||||
debug(
|
||||
`setOldReplicaList found ${replicatedVms.length} replica, and will delete ${this.#oldVmReplicaList.length} (retention ${settings.copyRetention}) for vm ${this.#props.vmUuid}`
|
||||
)
|
||||
}
|
||||
|
||||
async deleteOldReplicas() {
|
||||
debug(`deleteOldReplicas will delete ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
// destroy the VM on the right xapi
|
||||
await asyncMapSettled(this.#oldVmReplicaList, async vm => {
|
||||
debug('will delete old replica', vm.name_label)
|
||||
await vm.$xapi.VM_destroy(vm.$ref)
|
||||
})
|
||||
debug(`deleteOldReplicas deleted ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
}
|
||||
|
||||
async setupWriters() {
|
||||
debug('Setup writers ')
|
||||
const settings = this.#props.settings
|
||||
const allSettings = this.#props.job.settings
|
||||
|
||||
this.#storageRepositories.forEach(storageRepository => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[storageRepository.uuid],
|
||||
skipDeleteOldEntries: true, // delete is handled globally
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
const writer = new this.#ReplicationWriter({
|
||||
...this.#props,
|
||||
sr: storageRepository,
|
||||
settings: targetSettings,
|
||||
})
|
||||
this.#writers.push(writer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,3 @@
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
|
||||
export class AbstractWriter {
|
||||
constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
|
||||
this._config = config
|
||||
@@ -16,14 +13,4 @@ export class AbstractWriter {
|
||||
afterBackup() {}
|
||||
|
||||
healthCheck(sr) {}
|
||||
|
||||
_isAlreadyTransferred(timestamp) {
|
||||
const vmUuid = this._vmUuid
|
||||
const adapter = this._adapter
|
||||
const backupDir = getVmBackupDir(vmUuid)
|
||||
try {
|
||||
const actualMetadata = JSON.parse(adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`))
|
||||
return actualMetadata
|
||||
} catch (error) {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { ImportVmBackup } from '../../ImportVmBackup.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import * as MergeWorker from '../../merge-worker/index.mjs'
|
||||
import ms from 'ms'
|
||||
import { getEntryStatus } from '../../_getOldEntries.mjs'
|
||||
|
||||
const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
|
||||
|
||||
@@ -47,6 +48,19 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
}
|
||||
}
|
||||
|
||||
async getLongTermRetentionTags(currentEntry) {
|
||||
const settings = this._settings
|
||||
const scheduleId = this._scheduleId
|
||||
const vmUuid = this._vmUuid
|
||||
const adapter = this._adapter
|
||||
|
||||
const entries = await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)
|
||||
entries.push(currentEntry)
|
||||
const buckets = getEntryStatus(entries, currentEntry, settings.longTermRetention, settings.timezone)
|
||||
|
||||
return buckets.map(({ duration, dateBucket }) => `${duration}=${dateBucket}`)
|
||||
}
|
||||
|
||||
async beforeBackup() {
|
||||
const { handler } = this._adapter
|
||||
const vmBackupDir = this._vmBackupDir
|
||||
@@ -116,7 +130,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
xapi,
|
||||
}).run()
|
||||
} finally {
|
||||
await xapi.VM_destroy(restoredVm.$ref)
|
||||
await xapi.VM_destroy(restoredVm.$ref, { bypassBlockedOperation: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -72,7 +72,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
|
||||
xapi,
|
||||
}).run()
|
||||
} finally {
|
||||
healthCheckVmRef && (await xapi.VM_destroy(healthCheckVmRef))
|
||||
healthCheckVmRef && (await xapi.VM_destroy(healthCheckVmRef, { bypassBlockedOperation: true }))
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -5,10 +5,18 @@ const getReplicatedVmDatetime = vm => {
|
||||
return datetime
|
||||
}
|
||||
|
||||
const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
|
||||
export const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {string} scheduleOrJobId
|
||||
* @param {string} srUuid
|
||||
* @param {string} vmUuid
|
||||
* @returns {Array<XoVm>}
|
||||
*/
|
||||
export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
|
||||
const { all } = xapi.objects
|
||||
const all = xapi.objects.indexes.type.VM
|
||||
const vms = {}
|
||||
for (const key in all) {
|
||||
const object = all[key]
|
||||
@@ -20,9 +28,7 @@ export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
|
||||
'start' in object.blocked_operations &&
|
||||
(oc[JOB_ID] === scheduleOrJobId || oc[SCHEDULE_ID] === scheduleOrJobId) &&
|
||||
oc[REPLICATED_TO_SR_UUID] === srUuid &&
|
||||
(oc[VM_UUID] === vmUuid ||
|
||||
// 2018-03-28, JFT: to catch VMs replicated before this fix
|
||||
oc[VM_UUID] === undefined)
|
||||
oc[VM_UUID] === vmUuid
|
||||
) {
|
||||
vms[object.$id] = object
|
||||
}
|
||||
|
||||
317
@xen-orchestra/backups/disks/MergeRemoteDisk.mjs
Normal file
317
@xen-orchestra/backups/disks/MergeRemoteDisk.mjs
Normal file
@@ -0,0 +1,317 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import assert from 'assert'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { basename, dirname } from 'path'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
// @ts-ignore
|
||||
const { warn } = createLogger('remote-disk:merge')
|
||||
|
||||
/**
|
||||
* @typedef {Object} MergeState
|
||||
* @property {{ uuid: string }} child
|
||||
* @property {{ uuid: string }} parent
|
||||
* @property {number} currentBlock
|
||||
* @property {number} mergedDataSize
|
||||
* @property {'mergeBlocks' | 'cleanup'} step
|
||||
* @property {number} diskSize
|
||||
*/
|
||||
|
||||
export class MergeRemoteDisk {
|
||||
/**
|
||||
* @type {MergeState}
|
||||
*/
|
||||
#state = {
|
||||
child: { uuid: '0' },
|
||||
parent: { uuid: '0' },
|
||||
currentBlock: 0,
|
||||
mergedDataSize: 0,
|
||||
step: 'mergeBlocks',
|
||||
diskSize: 0,
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#statePath = ''
|
||||
|
||||
/**
|
||||
* @type {boolean}
|
||||
*/
|
||||
#isResuming
|
||||
|
||||
/**
|
||||
* @type {Logger | Function}
|
||||
*/
|
||||
#logInfo
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#mergeBlockConcurrency
|
||||
|
||||
/**
|
||||
* @type {Function}
|
||||
*/
|
||||
#onProgress
|
||||
|
||||
/**
|
||||
* @type {boolean}
|
||||
*/
|
||||
#removeUnused
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#writeStateDelay
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#lastStateWrittenAt
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @param {FileAccessor} handler
|
||||
* @param {Object} params
|
||||
* @param {Function} params.onProgress
|
||||
* @param {Logger | Function} params.logInfo
|
||||
* @param {boolean} params.removeUnused
|
||||
* @param {number} params.mergeBlockConcurrency
|
||||
* @param {number} params.writeStateDelay
|
||||
*/
|
||||
constructor(
|
||||
handler,
|
||||
{
|
||||
onProgress = () => {},
|
||||
logInfo = () => {},
|
||||
removeUnused = false,
|
||||
mergeBlockConcurrency = 2,
|
||||
writeStateDelay = 10e3,
|
||||
}
|
||||
) {
|
||||
this.#handler = handler
|
||||
this.#logInfo = logInfo
|
||||
this.#onProgress = onProgress
|
||||
this.#removeUnused = removeUnused
|
||||
this.#mergeBlockConcurrency = mergeBlockConcurrency
|
||||
this.#writeStateDelay = writeStateDelay
|
||||
|
||||
this.#isResuming = false
|
||||
this.#lastStateWrittenAt = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @returns {Promise<boolean>} isResuming
|
||||
*/
|
||||
async isResuming(parentDisk) {
|
||||
try {
|
||||
await this.#handler.readFile(
|
||||
dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async merge(parentDisk, childDisk) {
|
||||
this.#statePath = dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
|
||||
|
||||
try {
|
||||
const mergeStateContent = await this.#handler.readFile(this.#statePath)
|
||||
this.#state = JSON.parse(mergeStateContent)
|
||||
|
||||
// work-around a bug introduce in 97d94b795
|
||||
//
|
||||
// currentBlock could be `null` due to the JSON.stringify of a `NaN` value
|
||||
if (this.#state?.currentBlock === null) this.#state.currentBlock = 0
|
||||
this.#isResuming = true
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code !== 'ENOENT') {
|
||||
warn('problem while checking the merge state', { error })
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
/* eslint-disable no-fallthrough */
|
||||
switch (this.#state?.step ?? 'mergeBlocks') {
|
||||
case 'mergeBlocks':
|
||||
await this.#step_mergeBlocks(parentDisk, childDisk)
|
||||
case 'cleanup':
|
||||
await this.#step_cleanup(parentDisk, childDisk)
|
||||
return this.#cleanup(parentDisk, childDisk, true)
|
||||
default:
|
||||
warn(`Step ${this.#state?.step} is unknown`, { state: this.#state })
|
||||
}
|
||||
/* eslint-enable no-fallthrough */
|
||||
} catch (error) {
|
||||
await this.#cleanup(parentDisk, childDisk, false)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async #writeState() {
|
||||
try {
|
||||
await this.#handler.writeFile(this.#statePath, JSON.stringify(this.#state), { flags: 'w' })
|
||||
} catch (err) {
|
||||
warn('failed to write merge state', { error: err })
|
||||
}
|
||||
}
|
||||
|
||||
async #writeStateThrottled() {
|
||||
const now = Date.now()
|
||||
if (now - this.#lastStateWrittenAt > this.#writeStateDelay) {
|
||||
this.#lastStateWrittenAt = now
|
||||
await this.#writeState()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #step_mergeBlocks(parentDisk, childDisk) {
|
||||
const getMaxBlockCount = childDisk.getMaxBlockCount()
|
||||
|
||||
if (this.#isResuming) {
|
||||
const alreadyMergedBlocks = []
|
||||
for (let blockId = 0; blockId < this.#state.currentBlock; blockId++) {
|
||||
if (childDisk.hasBlock(blockId)) {
|
||||
alreadyMergedBlocks.push(blockId)
|
||||
}
|
||||
}
|
||||
|
||||
parentDisk.setAllocatedBlocks(alreadyMergedBlocks)
|
||||
} else {
|
||||
this.#state.child = { uuid: childDisk.getUuid() ?? 0 }
|
||||
this.#state.parent = { uuid: parentDisk.getUuid() ?? 0 }
|
||||
|
||||
// Finds first allocated block for the 2 following loops
|
||||
while (this.#state.currentBlock < getMaxBlockCount && !childDisk.hasBlock(this.#state.currentBlock)) {
|
||||
++this.#state.currentBlock
|
||||
}
|
||||
await this.#writeState()
|
||||
}
|
||||
|
||||
await this.#mergeBlocks(parentDisk, childDisk)
|
||||
await parentDisk.flushMetadata(childDisk)
|
||||
parentDisk.mergeMetadata(childDisk)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #mergeBlocks(parentDisk, childDisk) {
|
||||
this.#mergeBlockConcurrency =
|
||||
(await parentDisk.canMergeConcurently()) && (await childDisk.canMergeConcurently())
|
||||
? this.#mergeBlockConcurrency
|
||||
: 1
|
||||
|
||||
const toMerge = []
|
||||
for (let block = this.#state.currentBlock; block < childDisk.getMaxBlockCount(); block++) {
|
||||
if (childDisk.hasBlock(block)) {
|
||||
toMerge.push(block)
|
||||
}
|
||||
}
|
||||
|
||||
const nBlocks = toMerge.length
|
||||
this.#onProgress({ total: nBlocks, done: 0 })
|
||||
|
||||
const merging = new Set()
|
||||
let counter = 0
|
||||
await asyncEach(
|
||||
toMerge,
|
||||
async blockId => {
|
||||
merging.add(blockId)
|
||||
|
||||
const blockSize = await parentDisk.mergeBlock(childDisk, blockId, this.#isResuming)
|
||||
this.#state.mergedDataSize += blockSize
|
||||
|
||||
this.#state.currentBlock = Math.min(...merging) - 1
|
||||
merging.delete(blockId)
|
||||
|
||||
this.#onProgress({ total: nBlocks, done: counter + 1 })
|
||||
counter++
|
||||
await this.#writeStateThrottled()
|
||||
},
|
||||
{ concurrency: this.#mergeBlockConcurrency }
|
||||
)
|
||||
|
||||
await this.#writeState()
|
||||
|
||||
this.#state.diskSize = childDisk.getSizeOnDisk()
|
||||
|
||||
this.#onProgress({ total: nBlocks, done: nBlocks })
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #step_cleanup(parentDisk, childDisk) {
|
||||
assert.notEqual(this.#state, undefined)
|
||||
this.#state.step = 'cleanup'
|
||||
await this.#writeState()
|
||||
|
||||
const mergeTargetPath = childDisk.getPath()
|
||||
|
||||
// delete intermediate children if needed
|
||||
if (this.#removeUnused) {
|
||||
await childDisk.unlink()
|
||||
}
|
||||
|
||||
try {
|
||||
await parentDisk.rename(mergeTargetPath)
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code === 'ENOENT' && this.#isResuming) {
|
||||
// @ts-ignore
|
||||
this.#logInfo(`the parent disk was already renamed`, {
|
||||
parent: parentDisk.getPath(),
|
||||
mergeTarget: mergeTargetPath,
|
||||
})
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {boolean} cleanStateFile
|
||||
*
|
||||
* @returns {Promise<{mergedDataSize: number, finalDiskSize: number}>} result
|
||||
*/
|
||||
async #cleanup(parentDisk, childDisk, cleanStateFile) {
|
||||
const finalDiskSize = this.#state?.diskSize ?? 0
|
||||
const mergedDataSize = this.#state?.mergedDataSize ?? 0
|
||||
await parentDisk.close().catch(warn)
|
||||
await childDisk.close().catch(warn)
|
||||
|
||||
if (cleanStateFile) {
|
||||
await this.#handler.unlink(this.#statePath).catch(warn)
|
||||
}
|
||||
|
||||
return { mergedDataSize, finalDiskSize }
|
||||
}
|
||||
}
|
||||
204
@xen-orchestra/backups/disks/RemoteDisk.mjs
Normal file
204
@xen-orchestra/backups/disks/RemoteDisk.mjs
Normal file
@@ -0,0 +1,204 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
*/
|
||||
|
||||
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
export class RemoteDisk extends RandomAccessDisk {
|
||||
/**
|
||||
* Abstract
|
||||
* @param {Object} options
|
||||
* @param {boolean} options.force
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options = { force: false }) {
|
||||
throw new Error(`init must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
throw new Error(`close must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
throw new Error(`getVirtualSize must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
throw new Error(`getSizeOnDisk must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
throw new Error(`getBlockSize must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
throw new Error(`getPath must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
throw new Error(`getUuid must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
throw new Error(`canMergeConcurently must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {number} getMaxBlockCount
|
||||
*/
|
||||
getMaxBlockCount() {
|
||||
throw new Error(`getMaxBlockCount must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
throw new Error(`hasBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
throw new Error(`getBlockIndexes must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
throw new Error(`instantiateParent must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Writes a full block.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
throw new Error(`writeBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
throw new Error(`readBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
throw new Error(`mergeBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Manually set the disk allocated blocks.
|
||||
* @param {Array<number>} blockIds
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async setAllocatedBlocks(blockIds) {
|
||||
throw new Error(`setAllocatedBlocks must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
throw new Error(`flushMetadata must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
mergeMetadata(childDisk) {
|
||||
throw new Error(`mergeMetadata must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
throw new Error(`isDifferencing must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
throw new Error(`rename must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Deletes alias/disk/disks
|
||||
*/
|
||||
async unlink() {
|
||||
throw new Error(`unlink must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
async openParent() {
|
||||
const parent = await super.openParent()
|
||||
if (!(parent instanceof RemoteDisk)) {
|
||||
throw new Error('parent of a RemoteDisk must be also a RemoteDisk')
|
||||
}
|
||||
return parent
|
||||
}
|
||||
}
|
||||
894
@xen-orchestra/backups/disks/RemoteDisk.test.mjs
Normal file
894
@xen-orchestra/backups/disks/RemoteDisk.test.mjs
Normal file
@@ -0,0 +1,894 @@
|
||||
import { test } from 'node:test'
|
||||
import { strict as assert } from 'node:assert'
|
||||
|
||||
import tmp from 'tmp'
|
||||
import fs from 'fs-extra'
|
||||
import * as uuid from 'uuid'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { pFromCallback } from 'promise-toolbox'
|
||||
import { VHDFOOTER, VHDHEADER } from '../tests.fixtures.mjs'
|
||||
import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
|
||||
import { dirname, basename } from 'node:path'
|
||||
import { rimraf } from 'rimraf'
|
||||
import { MergeRemoteDisk } from './MergeRemoteDisk.mjs'
|
||||
|
||||
const { beforeEach, afterEach, describe } = test
|
||||
|
||||
let tempDir, handler, jobId, vdiId, basePath, relativePath
|
||||
const rootPath = 'xo-vm-backups/VMUUID'
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
handler = getHandler({ url: `file://${tempDir}` })
|
||||
await handler.sync()
|
||||
jobId = uniqueId()
|
||||
vdiId = uniqueId()
|
||||
relativePath = `vdis/${jobId}/${vdiId}`
|
||||
basePath = `${rootPath}/${relativePath}`
|
||||
await fs.mkdirp(`${tempDir}/${basePath}`)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rimraf(tempDir)
|
||||
await handler.forget()
|
||||
})
|
||||
|
||||
const uniqueId = () => uuid.v1()
|
||||
const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
|
||||
|
||||
async function generateVhd(path, opts = {}) {
|
||||
let vhd
|
||||
|
||||
let dataPath = path
|
||||
if (opts.useAlias) {
|
||||
await handler.mkdir(dirname(path) + '/data/')
|
||||
dataPath = dirname(path) + '/data/' + basename(path)
|
||||
}
|
||||
if (opts.mode === 'directory') {
|
||||
await handler.mkdir(dataPath)
|
||||
vhd = new VhdDirectory(handler, dataPath)
|
||||
} else {
|
||||
const fd = await handler.openFile(dataPath, 'wx')
|
||||
vhd = new VhdFile(handler, fd)
|
||||
}
|
||||
|
||||
vhd.header = { ...VHDHEADER, ...opts.header }
|
||||
vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
|
||||
|
||||
if (vhd.header.parentUuid) {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
|
||||
} else {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
|
||||
}
|
||||
|
||||
if (opts.useAlias === true) {
|
||||
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
|
||||
}
|
||||
|
||||
if (opts.blocks) {
|
||||
for (const blockId of opts.blocks) {
|
||||
await vhd.writeEntireBlock({ id: blockId, buffer: Buffer.alloc(2 * 1024 * 1024 + 512, blockId) })
|
||||
}
|
||||
}
|
||||
await vhd.writeBlockAllocationTable()
|
||||
await vhd.writeHeader()
|
||||
await vhd.writeFooter()
|
||||
return vhd
|
||||
}
|
||||
|
||||
/**
|
||||
* RemoteVhdDisk
|
||||
*/
|
||||
describe('tests RemoteVhdDisk', { concurrency: 1 }, () => {
|
||||
test('RemoteVhdDisk should read block', async () => {
|
||||
// Create VHD disk
|
||||
await generateVhd(`${basePath}/disk.vhd`, { blocks: [0, 1] })
|
||||
|
||||
const disk = new RemoteVhdDisk({ handler, path: `${basePath}/disk.vhd` })
|
||||
|
||||
await disk.init({ force: false })
|
||||
|
||||
const readBlock = await disk.readBlock(1)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDisk should write block', async () => {
|
||||
// Create VHD disk
|
||||
await generateVhd(`${basePath}/disk.vhd`, { blocks: [0, 1] })
|
||||
|
||||
const disk = new RemoteVhdDisk({ handler, path: `${basePath}/disk.vhd` })
|
||||
|
||||
await disk.init({ force: false })
|
||||
|
||||
const blockId = 2
|
||||
await disk.writeBlock({ index: blockId, data: Buffer.alloc(2 * 1024 * 1024, blockId) })
|
||||
const readBlock = await disk.readBlock(blockId)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDisk should read block on VHD directory', async () => {
|
||||
// Create VHD disk
|
||||
await generateVhd(`${basePath}/directory.vhd`, { blocks: [0, 1], mode: 'directory', useAlias: true })
|
||||
|
||||
const directory = new RemoteVhdDisk({ handler, path: `${basePath}/directory.vhd.alias.vhd` })
|
||||
|
||||
await directory.init({ force: false })
|
||||
|
||||
const readBlock = await directory.readBlock(1)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDisk should write block on VHD directory', async () => {
|
||||
// Create VHD disk
|
||||
await generateVhd(`${basePath}/directory.vhd`, { blocks: [0, 1], mode: 'directory', useAlias: true })
|
||||
|
||||
const directory = new RemoteVhdDisk({ handler, path: `${basePath}/directory.vhd.alias.vhd` })
|
||||
|
||||
await directory.init({ force: false })
|
||||
|
||||
const blockId = 2
|
||||
await directory.writeBlock({ index: blockId, data: Buffer.alloc(2 * 1024 * 1024, blockId) })
|
||||
const readBlock = await directory.readBlock(blockId)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDisk should not support non alias VHD directory', async () => {
|
||||
// Create VHD directory
|
||||
await generateVhd(`${basePath}/directory.vhd`, { blocks: [0, 1], mode: 'directory' })
|
||||
|
||||
const directory = new RemoteVhdDisk({ handler, path: `${basePath}/directory.vhd` })
|
||||
|
||||
try {
|
||||
await directory.init({ force: true })
|
||||
} catch (err) {
|
||||
assert.strictEqual(
|
||||
err.code,
|
||||
'NOT_SUPPORTED',
|
||||
'Initializing a vhd directory without using alias should raise a not supported error'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
assert.strictEqual(
|
||||
true,
|
||||
false,
|
||||
'Initializing a vhd directory without using alias should raise a not supported error'
|
||||
)
|
||||
})
|
||||
|
||||
test('RemoteVhdDisk closes disk on init error', async () => {
|
||||
// Create VHD disk
|
||||
await generateVhd(`${basePath}/disk.vhd`, { blocks: [0] })
|
||||
|
||||
const disk = new RemoteVhdDisk({ handler, path: `${basePath}/disk.vhd` })
|
||||
|
||||
// Override the disk close method to monitor its execution
|
||||
let diskClosed = false
|
||||
const originalClose = disk.close.bind(disk)
|
||||
disk.close = async () => {
|
||||
diskClosed = true
|
||||
await originalClose()
|
||||
}
|
||||
|
||||
// Make isDirectory throw an error to simulate init failure
|
||||
disk.isDirectory = async () => {
|
||||
throw new Error('simulated write error')
|
||||
}
|
||||
|
||||
// Run init and expect it to throw
|
||||
let threw = false
|
||||
try {
|
||||
await disk.init()
|
||||
} catch (err) {
|
||||
threw = true
|
||||
assert.equal(err.message, 'simulated write error')
|
||||
}
|
||||
|
||||
assert.equal(threw, true, 'merge should throw')
|
||||
assert.equal(diskClosed, true, 'disk disk should be closed on error')
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* RemoteVhdDiskChain
|
||||
*/
|
||||
describe('tests RemoteVhdDiskChain', { concurrency: 1 }, () => {
|
||||
test('RemoteVhdDiskChain should read block', async () => {
|
||||
// Create parent and child VHDs
|
||||
const chainParent = await generateVhd(`${basePath}/parent.vhd`, { blocks: [0] })
|
||||
const chainChild = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'parent.vhd',
|
||||
parentUuid: chainParent.footer.uuid,
|
||||
},
|
||||
blocks: [1],
|
||||
})
|
||||
await generateVhd(`${basePath}/child_2.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child.vhd',
|
||||
parentUuid: chainChild.footer.uuid,
|
||||
},
|
||||
blocks: [1],
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/parent.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
const child_2 = new RemoteVhdDisk({ handler, path: `${basePath}/child_2.vhd` })
|
||||
const diskChain = new RemoteVhdDiskChain({ disks: [parent, child, child_2] })
|
||||
|
||||
await diskChain.init({ force: false })
|
||||
|
||||
const readBlock = await diskChain.readBlock(1)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDiskChain should read block on VHD directory', async () => {
|
||||
// Create parent and child VHD directories
|
||||
const chainParent = await generateVhd(`${basePath}/parent.vhd`, { blocks: [0], mode: 'directory', useAlias: true })
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'parent.vhd',
|
||||
parentUuid: chainParent.footer.uuid,
|
||||
},
|
||||
blocks: [1],
|
||||
mode: 'directory',
|
||||
useAlias: true,
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/parent.vhd.alias.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
const directoryChain = new RemoteVhdDiskChain({ disks: [parent, child] })
|
||||
|
||||
await directoryChain.init({ force: false })
|
||||
|
||||
const readBlock = await directoryChain.readBlock(1)
|
||||
|
||||
assert.equal(readBlock.data.length, 2097152, 'block should be readable')
|
||||
})
|
||||
|
||||
test('RemoteVhdDiskChain should closes disks on init error', async () => {
|
||||
// Create parent and child VHDs
|
||||
const chainParent = await generateVhd(`${basePath}/parent.vhd`, { blocks: [0] })
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'parent.vhd',
|
||||
parentUuid: chainParent.footer.uuid,
|
||||
},
|
||||
blocks: [1],
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/parent.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
const diskChain = new RemoteVhdDiskChain({ disks: [parent, child] })
|
||||
|
||||
// Override the disks close method to monitor their execution
|
||||
let parentClosed = false
|
||||
const originalParentClose = parent.close.bind(parent)
|
||||
parent.close = async () => {
|
||||
parentClosed = true
|
||||
await originalParentClose()
|
||||
}
|
||||
let childClosed = false
|
||||
const originalChildClose = child.close.bind(child)
|
||||
child.close = async () => {
|
||||
childClosed = true
|
||||
await originalChildClose()
|
||||
}
|
||||
|
||||
// Make isDifferencing throw an error to simulate init failure of the child disk
|
||||
child.isDifferencing = () => {
|
||||
throw new Error('simulated write error')
|
||||
}
|
||||
|
||||
// Run init and expect it to throw
|
||||
let threw = false
|
||||
try {
|
||||
await diskChain.init({ force: false })
|
||||
} catch (err) {
|
||||
threw = true
|
||||
assert.equal(err.message, 'simulated write error')
|
||||
}
|
||||
|
||||
assert.equal(threw, true, 'merge should throw')
|
||||
assert.equal(parentClosed, true, 'parent disk should be closed on error')
|
||||
assert.equal(childClosed, true, 'child disk should be closed on error')
|
||||
})
|
||||
|
||||
test('RemoteVhdDiskChain should not support non differencing VHD chain children', async () => {
|
||||
// Create VHD chain
|
||||
await generateVhd(`${basePath}/disk_1.vhd`, { blocks: [2] })
|
||||
await generateVhd(`${basePath}/disk_2.vhd`, { blocks: [3] })
|
||||
|
||||
const disk1 = new RemoteVhdDisk({ handler, path: `${basePath}/disk_1.vhd` })
|
||||
const disk2 = new RemoteVhdDisk({ handler, path: `${basePath}/disk_2.vhd` })
|
||||
const diskChain = new RemoteVhdDiskChain({ disks: [disk1, disk2] })
|
||||
|
||||
try {
|
||||
await diskChain.init({ force: true })
|
||||
} catch (err) {
|
||||
assert.strictEqual(err.code, 'NOT_SUPPORTED', "Can't init vhd directory with non differencing child disks")
|
||||
return
|
||||
}
|
||||
assert.strictEqual(true, false, "Can't init vhd directory with non differencing child disks")
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* MergeVhdChain
|
||||
*/
|
||||
describe('tests MergeVhdChain', { concurrency: 1 }, () => {
|
||||
test('mergeVhdChain merges a simple ancestor VHD + child VHD', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3] })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, child)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child.vhd'])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges a simple ancestor VHD + child VHD chain', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and children VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
|
||||
const chainChild1 = await generateVhd(`${basePath}/child_1.vhd`, { blocks: [2] })
|
||||
const chainChild2 = await generateVhd(`${basePath}/child_2.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child_1.vhd',
|
||||
parentUuid: chainChild1.footer.uuid,
|
||||
},
|
||||
blocks: [3],
|
||||
})
|
||||
await generateVhd(`${basePath}/child_3.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child_2.vhd',
|
||||
parentUuid: chainChild2.footer.uuid,
|
||||
},
|
||||
blocks: [4],
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child1 = new RemoteVhdDisk({ handler, path: `${basePath}/child_1.vhd` })
|
||||
const child2 = new RemoteVhdDisk({ handler, path: `${basePath}/child_2.vhd` })
|
||||
const child3 = new RemoteVhdDisk({ handler, path: `${basePath}/child_3.vhd` })
|
||||
const childDiskChain = new RemoteVhdDiskChain({ disks: [child1, child2, child3] })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child1.init({ force: isResumingMerge })
|
||||
await child2.init({ force: isResumingMerge })
|
||||
await child3.init({ force: isResumingMerge })
|
||||
await childDiskChain.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent
|
||||
.getBlockIndexes()
|
||||
.concat(child1.getBlockIndexes())
|
||||
.concat(child2.getBlockIndexes())
|
||||
.concat(child3.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, childDiskChain)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child_3.vhd']) // TODO: Check that it's the correct child that is remaining.
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges an alias ancestor VHD + alias child VHD', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1], useAlias: true })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3], useAlias: true })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: true })
|
||||
await child.init({ force: true })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, child)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child.vhd.alias.vhd', 'data'])
|
||||
|
||||
const remainingDatas = await handler.list(basePath + '/data')
|
||||
assert.deepEqual(remainingDatas, ['ancestor.vhd'])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges an alias ancestor VHD + alias child VHD chain', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor VHD
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1], useAlias: true })
|
||||
|
||||
// Create child VHD chain
|
||||
const chainParent = await generateVhd(`${basePath}/child_1.vhd`, { blocks: [2], useAlias: true })
|
||||
await generateVhd(`${basePath}/child_2.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child_1.vhd',
|
||||
parentUuid: chainParent.footer.uuid,
|
||||
},
|
||||
blocks: [3],
|
||||
useAlias: true,
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child1 = new RemoteVhdDisk({ handler, path: `${basePath}/child_1.vhd.alias.vhd` })
|
||||
const child2 = new RemoteVhdDisk({ handler, path: `${basePath}/child_2.vhd.alias.vhd` })
|
||||
const childDiskChain = new RemoteVhdDiskChain({ disks: [child1, child2] })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child1.init({ force: isResumingMerge })
|
||||
await child2.init({ force: isResumingMerge })
|
||||
await childDiskChain.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child1.getBlockIndexes()).concat(child2.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, childDiskChain)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child_2.vhd.alias.vhd', 'data'])
|
||||
|
||||
const remainingDatas = await handler.list(basePath + '/data')
|
||||
assert.deepEqual(remainingDatas, ['ancestor.vhd'])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges an alias ancestor VHD + child VHD', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1], useAlias: true })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3] })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, child)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child.vhd', 'data'])
|
||||
|
||||
const remainingDatas = await handler.list(basePath + '/data')
|
||||
assert.deepEqual(remainingDatas, ['ancestor.vhd'])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges an ancestor VHD + alias child VHD', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3], useAlias: true })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, child)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child.vhd.alias.vhd', 'data'])
|
||||
|
||||
const remainingDatas = await handler.list(basePath + '/data')
|
||||
assert.deepEqual(remainingDatas, [])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeVhdChain merges a alias ancestor VHD directory + alias child VHD directory', async () => {
|
||||
let progressCalls = 0
|
||||
const onProgress = () => {
|
||||
progressCalls++
|
||||
}
|
||||
|
||||
// Create ancestor and child VHD directories
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1], mode: 'directory', useAlias: true })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3], mode: 'directory', useAlias: true })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { onProgress, removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parent, child)
|
||||
|
||||
assert.ok(result.finalDiskSize > 0, 'merged disks should have non-zero size')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
assert.ok(progressCalls > 0, 'onProgress should have been called')
|
||||
|
||||
// Check that ancestor was renamed to child
|
||||
const remainingDisks = await handler.list(basePath)
|
||||
assert.deepEqual(remainingDisks, ['child.vhd.alias.vhd', 'data'])
|
||||
|
||||
const remainingDatas = await handler.list(basePath + '/data')
|
||||
assert.deepEqual(remainingDatas, ['ancestor.vhd'])
|
||||
|
||||
const parentIndexes = parent.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
|
||||
const ancestorData = await handler.list(`${basePath}/data/ancestor.vhd`)
|
||||
assert.ok(ancestorData.length > 0, 'ancestor.vhd directory should contain blocks')
|
||||
})
|
||||
|
||||
test('mergeVhdChain closes disks on error', async () => {
|
||||
// Create parent and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0] })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [1] })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { removeUnused: true })
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
// Override the disks close method to monitor their execution
|
||||
let parentClosed = false
|
||||
let childClosed = false
|
||||
const originalParentClose = parent.close.bind(parent)
|
||||
parent.close = async () => {
|
||||
parentClosed = true
|
||||
await originalParentClose()
|
||||
}
|
||||
const originalChildClose = child.close.bind(child)
|
||||
child.close = async () => {
|
||||
childClosed = true
|
||||
await originalChildClose()
|
||||
}
|
||||
|
||||
// Make writeBlock throw an error to simulate failure
|
||||
parent.writeBlock = async () => {
|
||||
throw new Error('simulated write error')
|
||||
}
|
||||
|
||||
// Run merge and expect it to throw
|
||||
let threw = false
|
||||
try {
|
||||
await mergeRemoteDisk.merge(parent, child)
|
||||
} catch (err) {
|
||||
threw = true
|
||||
assert.equal(err.message, 'simulated write error')
|
||||
}
|
||||
|
||||
assert.equal(threw, true, 'merge should throw')
|
||||
assert.equal(parentClosed, true, 'parent disk should be closed on error')
|
||||
assert.equal(childClosed, true, 'child disk should be closed on error')
|
||||
})
|
||||
|
||||
test('mergeRemoteDisk resumes after interruption', async () => {
|
||||
// Create parent and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1] })
|
||||
await generateVhd(`${basePath}/child.vhd`, { blocks: [2, 3, 4, 5, 6, 7, 8, 9, 10] })
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, { removeUnused: true, writeStateDelay: 0 })
|
||||
|
||||
let isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
// Track write progress
|
||||
let blocksWritten = 0
|
||||
const originalWriteBlock = parent.writeBlock.bind(parent)
|
||||
|
||||
// Make writeBlock fail on the second block
|
||||
parent.writeBlock = async diskBlock => {
|
||||
if (blocksWritten === 5) {
|
||||
throw new Error('simulated interruption')
|
||||
}
|
||||
blocksWritten++
|
||||
return originalWriteBlock(diskBlock)
|
||||
}
|
||||
|
||||
// First merge attempt should throw
|
||||
let threw = false
|
||||
try {
|
||||
await mergeRemoteDisk.merge(parent, child)
|
||||
} catch (err) {
|
||||
threw = true
|
||||
assert.equal(err.message, 'simulated interruption')
|
||||
}
|
||||
assert.equal(threw, true, 'merge should throw on simulated interruption')
|
||||
|
||||
// Verify that state file exists
|
||||
assert.ok(
|
||||
(await handler.list(dirname(parent.getPath()))).includes(`.${basename(parent.getPath())}.merge.json`),
|
||||
'merge state file should exist after failure'
|
||||
)
|
||||
|
||||
// Recreate new remoteDisks to simulate a new operation
|
||||
const parent2 = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd` })
|
||||
const child2 = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd` })
|
||||
|
||||
isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(isResumingMerge, 'merge is resuming')
|
||||
|
||||
await parent2.init({ force: isResumingMerge })
|
||||
await child2.init({ force: isResumingMerge })
|
||||
|
||||
const originalWriteBlock2 = parent2.writeBlock.bind(parent2)
|
||||
|
||||
// Make writeBlock fail on the second block
|
||||
parent2.writeBlock = async diskBlock => {
|
||||
blocksWritten++
|
||||
return originalWriteBlock2(diskBlock)
|
||||
}
|
||||
|
||||
// Resume merge
|
||||
const result = await mergeRemoteDisk.merge(parent2, child2)
|
||||
|
||||
// Verify merge completed
|
||||
assert.ok(result.finalDiskSize > 0, 'final disk size should be > 0')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
|
||||
// Verify state file removed after successful resume
|
||||
assert.equal(
|
||||
(await handler.list(dirname(parent2.getPath()))).includes(`.${basename(parent2.getPath())}.merge.json`),
|
||||
false,
|
||||
'merge state file should be removed after successful resume'
|
||||
)
|
||||
|
||||
// Verify all blocks present in parent
|
||||
const parentIndexes = parent2.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
})
|
||||
|
||||
test('mergeRemoteDisk resumes after interruption with VHD directory', async () => {
|
||||
// Track write progress
|
||||
let blocksWritten = 0
|
||||
|
||||
// Create parent and child VHDs
|
||||
await generateVhd(`${basePath}/ancestor.vhd`, { blocks: [0, 1], useAlias: true, mode: 'directory' })
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
blocks: [2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
useAlias: true,
|
||||
mode: 'directory',
|
||||
})
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
|
||||
removeUnused: true,
|
||||
writeStateDelay: 0,
|
||||
mergeBlockConcurrency: 4,
|
||||
})
|
||||
|
||||
let isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(!isResumingMerge, 'merge is not resuming')
|
||||
|
||||
await parent.init({ force: isResumingMerge })
|
||||
await child.init({ force: isResumingMerge })
|
||||
|
||||
const expectedIndexes = parent.getBlockIndexes().concat(child.getBlockIndexes())
|
||||
|
||||
const originalMergeBlock = parent.mergeBlock.bind(parent)
|
||||
|
||||
// Make writeBlock fail on the second block
|
||||
parent.mergeBlock = async (disk, index, isResumingMerge) => {
|
||||
if (blocksWritten === 4) {
|
||||
throw new Error('simulated interruption')
|
||||
}
|
||||
blocksWritten++
|
||||
|
||||
return originalMergeBlock(disk, index, isResumingMerge)
|
||||
}
|
||||
|
||||
// First merge attempt should throw
|
||||
let threw = false
|
||||
try {
|
||||
await mergeRemoteDisk.merge(parent, child)
|
||||
} catch (err) {
|
||||
threw = true
|
||||
assert.equal(err.message, 'simulated interruption')
|
||||
}
|
||||
assert.equal(threw, true, 'merge should throw on simulated interruption')
|
||||
|
||||
// Verify that state file exists
|
||||
assert.ok(
|
||||
(await handler.list(dirname(parent.getPath()))).includes(`.${basename(parent.getPath())}.merge.json`),
|
||||
'merge state file should exist after failure'
|
||||
)
|
||||
|
||||
// Recreate new remoteDisks to simulate a new operation
|
||||
const parent2 = new RemoteVhdDisk({ handler, path: `${basePath}/ancestor.vhd.alias.vhd` })
|
||||
const child2 = new RemoteVhdDisk({ handler, path: `${basePath}/child.vhd.alias.vhd` })
|
||||
|
||||
isResumingMerge = await mergeRemoteDisk.isResuming(parent)
|
||||
|
||||
assert.ok(isResumingMerge, 'merge is resuming')
|
||||
|
||||
await parent2.init({ force: isResumingMerge })
|
||||
await child2.init({ force: isResumingMerge })
|
||||
|
||||
const originalWriteBlock2 = parent2.writeBlock.bind(parent2)
|
||||
|
||||
// Make writeBlock fail on the second block
|
||||
parent2.writeBlock = async diskBlock => {
|
||||
blocksWritten++
|
||||
return originalWriteBlock2(diskBlock)
|
||||
}
|
||||
|
||||
// Resume merge
|
||||
const result = await mergeRemoteDisk.merge(parent2, child2)
|
||||
|
||||
// Verify merge completed
|
||||
assert.ok(result.finalDiskSize > 0, 'final disk size should be > 0')
|
||||
assert.ok(result.mergedDataSize > 0, 'merged data size should be > 0')
|
||||
|
||||
// Verify state file removed after successful resume
|
||||
assert.equal(
|
||||
(await handler.list(dirname(parent2.getPath()))).includes(`.${basename(parent2.getPath())}.merge.json`),
|
||||
false,
|
||||
'merge state file should be removed after successful resume'
|
||||
)
|
||||
|
||||
// Verify all blocks present in parent BAT
|
||||
const parentIndexes = parent2.getBlockIndexes()
|
||||
assert.deepEqual(parentIndexes, expectedIndexes, 'all child blocks should be merged into parent')
|
||||
|
||||
// Verify all blocks actually present in parent
|
||||
for (const index of expectedIndexes) {
|
||||
const parentBlockPath = parent2.getBlockPath(index)
|
||||
|
||||
let size
|
||||
try {
|
||||
size = await handler.getSize(parentBlockPath)
|
||||
} catch {
|
||||
assert.fail(`block ${index} is missing from parent directory`)
|
||||
}
|
||||
|
||||
assert.ok(size >= parent2.getBlockSize(), `block ${index} complete`)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,171 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').Disk} Disk
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
*/
|
||||
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { DISK_TYPES } from 'vhd-lib/_constants.js'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
/**
|
||||
* Represents a remote VHD (Virtual Hard Disk) that extends RandomAccessDisk.
|
||||
*/
|
||||
export class RemoteVhd extends RandomAccessDisk {
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @type {VhdFile | VhdDirectory | undefined}
|
||||
*/
|
||||
#vhd
|
||||
|
||||
/**
|
||||
* @type {boolean | undefined}
|
||||
*/
|
||||
#isDifferencing
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
get path() {
|
||||
return this.#path
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getvirtualsize of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.footer.currentSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return 2 * 1024 * 1024
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init() {
|
||||
const { value, dispose } = await openVhd(this.#handler, this.#path)
|
||||
this.#vhd = value
|
||||
this.#dispose = dispose
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await this.#dispose()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call hasblock of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.containsBlock(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getBlockIndexes of a RemoteVhd before init`)
|
||||
}
|
||||
const index = []
|
||||
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
|
||||
if (this.hasBlock(blockIndex)) {
|
||||
index.push(blockIndex)
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>}
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhd before init`)
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @returns {RandomAccessDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call openParent of a RemoteVhd before init`)
|
||||
}
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
if (!parentPath) {
|
||||
throw new Error(`Disk ${this.#path} doesn't have parents`)
|
||||
}
|
||||
const parent = new RemoteVhd({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
if (this.#isDifferencing === undefined) {
|
||||
throw new Error(`can't call isDifferencing of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#isDifferencing
|
||||
}
|
||||
}
|
||||
459
@xen-orchestra/backups/disks/RemoteVhdDisk.mjs
Normal file
459
@xen-orchestra/backups/disks/RemoteVhdDisk.mjs
Normal file
@@ -0,0 +1,459 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
* @typedef {import('vhd-lib/_createFooterHeader').VhdFooter} VhdFooter
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*
|
||||
|
||||
*/
|
||||
|
||||
import { openVhd, VhdAbstract, VhdDirectory } from 'vhd-lib'
|
||||
import { RemoteDisk } from './RemoteDisk.mjs'
|
||||
import { DISK_TYPES } from 'vhd-lib/_constants.js'
|
||||
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
// import { set as setBitmap } from 'vhd-lib/_bitmap.js'
|
||||
import { stringify } from 'uuid'
|
||||
import { dirname, join } from 'node:path'
|
||||
|
||||
export class RemoteVhdDisk extends RemoteDisk {
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @type {VhdFile | VhdDirectory | undefined}
|
||||
*/
|
||||
#vhd
|
||||
|
||||
/**
|
||||
* @type {boolean | undefined}
|
||||
*/
|
||||
#isDifferencing
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#blockSize = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#bitmapSize = 512
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} [options]
|
||||
* @param {boolean} [options.force=false]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options = {}) {
|
||||
if (this.#vhd === undefined) {
|
||||
try {
|
||||
const { value, dispose } = await openVhd(this.#handler, await resolveVhdAlias(this.#handler, this.#path), {
|
||||
checkSecondFooter: !options.force,
|
||||
})
|
||||
this.#vhd = value
|
||||
|
||||
if ((await this.isDirectory()) && !isVhdAlias(this.#path)) {
|
||||
this.#vhd = undefined
|
||||
throw Object.assign(new Error("Can't init vhd directory without using alias"), { code: 'NOT_SUPPORTED' })
|
||||
}
|
||||
|
||||
this.#dispose = dispose
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
} catch (error) {
|
||||
await this.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await this.#dispose()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getVirtualSize of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#vhd.footer.currentSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getVirtualSize of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd.streamSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return this.#blockSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
return this.#path
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getUid of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return stringify(this.#vhd.footer.uuid)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getParentUuid() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getParentUid of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return stringify(this.#vhd.header.parentUuid)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
return await this.isDirectory()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} getMaxBlockCount
|
||||
*/
|
||||
getMaxBlockCount() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getMaxBlockCount of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd.header.maxTableEntries
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call hasblock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#vhd.containsBlock(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getBlockIndexes of a RemoteVhdDisk before init`)
|
||||
}
|
||||
const indexes = []
|
||||
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
|
||||
if (this.hasBlock(blockIndex)) {
|
||||
indexes.push(blockIndex)
|
||||
}
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call instantiateParent of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
|
||||
if (!parentPath) {
|
||||
throw new Error(`disk ${this.#path} doesn't have parents`)
|
||||
}
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a full block into this VHD.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
await this.#vhd.writeEntireBlock({
|
||||
id: diskBlock.index,
|
||||
buffer: Buffer.concat([Buffer.alloc(this.#bitmapSize, 255), diskBlock.data]),
|
||||
})
|
||||
|
||||
return this.getBlockSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
if ((await this.isDirectory()) && childDisk instanceof RemoteVhdDisk && (await childDisk.isDirectory())) {
|
||||
try {
|
||||
await this.#handler.rename(childDisk.getBlockPath(index), this.getBlockPath(index))
|
||||
|
||||
this.setAllocatedBlocks([index])
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code === 'ENOENT' && isResumingMerge === true) {
|
||||
// when resuming, the blocks moved since the last merge state write are
|
||||
// not in the child anymore but it should be ok
|
||||
|
||||
// it will throw an error if block is missing in parent
|
||||
// won't detect if the block was already in parent and is broken/missing in child
|
||||
|
||||
// since we can't know the initial size, this will create a discrepancy
|
||||
// on the size
|
||||
const { data } = await this.readBlock(index)
|
||||
if (data.length !== this.getBlockSize()) {
|
||||
throw error
|
||||
} else {
|
||||
this.setAllocatedBlocks([index])
|
||||
}
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return this.getBlockSize()
|
||||
} else {
|
||||
return this.writeBlock(await childDisk.readBlock(index))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a specific block path from the VHD directory.
|
||||
* @param {number} index
|
||||
* @returns {string} blockPath
|
||||
*/
|
||||
getBlockPath(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
if (this.#vhd instanceof VhdDirectory) {
|
||||
return this.#vhd.getFullBlockPath(index)
|
||||
} else {
|
||||
throw new Error(`can't call getBlockPath of non directory VHD`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually set the disk allocated blocks.
|
||||
* @param {Array<number>} blockIds
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async setAllocatedBlocks(blockIds) {
|
||||
if (this.#vhd instanceof VhdDirectory) {
|
||||
this.#vhd.setAllocatedBlocks(blockIds)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes Block Allocation Table
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call flushMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
await this.#vhd.writeBlockAllocationTable()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {VhdFooter}
|
||||
*/
|
||||
getMetadata() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd.footer
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
*/
|
||||
async mergeMetadata(childDisk) {
|
||||
const childDiskMetadata = childDisk.getMetadata()
|
||||
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call mergeMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.currentSize = childDiskMetadata.currentSize
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.diskGeometry = { ...childDiskMetadata.diskGeometry }
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.originalSize = childDiskMetadata.originalSize
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.timestamp = childDiskMetadata.timestamp
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.uuid = childDiskMetadata.uuid
|
||||
|
||||
await this.#vhd.writeFooter()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
if (this.#isDifferencing === undefined) {
|
||||
throw new Error(`can't call isDifferencing of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#isDifferencing
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
if (isVhdAlias(newPath)) {
|
||||
const dataPath = await resolveVhdAlias(this.#handler, this.#path)
|
||||
|
||||
await this.#handler.unlink(this.#path)
|
||||
await this.#handler.unlink(newPath)
|
||||
|
||||
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
|
||||
|
||||
this.#path = newPath
|
||||
} else {
|
||||
try {
|
||||
await this.#handler.unlink(newPath)
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(newPath).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
await this.#handler.rename(this.#path, newPath)
|
||||
|
||||
this.#path = newPath
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes disk
|
||||
*/
|
||||
async unlink() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
await this.close()
|
||||
|
||||
if (isVhdAlias(this.#path)) {
|
||||
try {
|
||||
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#handler.unlink(this.#path)
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(this.#path).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the disk is a VHD directory.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isDirectory() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call isDirectory of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd instanceof VhdDirectory
|
||||
}
|
||||
}
|
||||
271
@xen-orchestra/backups/disks/RemoteVhdDiskChain.mjs
Normal file
271
@xen-orchestra/backups/disks/RemoteVhdDiskChain.mjs
Normal file
@@ -0,0 +1,271 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import { RemoteDisk } from './RemoteDisk.mjs'
|
||||
|
||||
export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
/**
|
||||
* @type {RemoteVhdDisk[]}
|
||||
*/
|
||||
#disks
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#blockSize = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#headerSize = 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#footerSize = 512
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#bitmapSize = 512
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {RemoteVhdDisk[]} params.disks
|
||||
*/
|
||||
constructor({ disks }) {
|
||||
super()
|
||||
this.#disks = disks
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the VHD chain
|
||||
* @param {Object} options
|
||||
* @param {boolean} options.force
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options) {
|
||||
try {
|
||||
await Promise.all(this.#disks.map(disk => disk.init(options)))
|
||||
let parentUuid = ''
|
||||
for (const [index, disk] of this.#disks.entries()) {
|
||||
if (index !== 0) {
|
||||
if (!disk.isDifferencing()) {
|
||||
throw Object.assign(new Error("Can't init vhd directory with non differencing child disks"), {
|
||||
code: 'NOT_SUPPORTED',
|
||||
})
|
||||
}
|
||||
if (disk.getParentUuid() !== parentUuid) {
|
||||
throw Object.assign(new Error("Can't init vhd directory with incorrect parentage"), {
|
||||
code: 'NOT_SUPPORTED',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
parentUuid = disk.getUuid()
|
||||
}
|
||||
} catch (error) {
|
||||
await this.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await Promise.all(this.#disks.map(disk => disk.close()))
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
return this.#disks[this.#disks.length - 1].getVirtualSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
const batEntrySize = 4
|
||||
const sectorSize = 512
|
||||
const batSize = Math.ceil((this.getMaxBlockCount() * batEntrySize) / sectorSize) * sectorSize
|
||||
|
||||
return (
|
||||
this.#footerSize +
|
||||
this.#headerSize +
|
||||
batSize +
|
||||
this.getBlockIndexes().length * (this.#blockSize + this.#bitmapSize) +
|
||||
this.#footerSize
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return this.#disks[this.#disks.length - 1].getBlockSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
return this.#disks[this.#disks.length - 1].getPath()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
return this.#disks[this.#disks.length - 1].getUuid()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
for (const disk of this.#disks) {
|
||||
if (!(await disk.isDirectory())) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} getMaxBlockCount
|
||||
*/
|
||||
getMaxBlockCount() {
|
||||
return this.#disks[this.#disks.length - 1].getMaxBlockCount()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
for (const disk of this.#disks) {
|
||||
if (disk.hasBlock(index)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
const indexes = new Set()
|
||||
for (const disk of this.#disks) {
|
||||
for (const index of disk.getBlockIndexes()) {
|
||||
indexes.add(index)
|
||||
}
|
||||
}
|
||||
return [...indexes]
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
return this.#disks[0].instantiateParent()
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a full block into this VHD.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
throw new Error(`Can't write blocks into a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
const reversedDisks = this.#disks.slice().reverse()
|
||||
for (const disk of reversedDisks) {
|
||||
if (disk.hasBlock(index)) {
|
||||
return disk.readBlock(index)
|
||||
}
|
||||
}
|
||||
throw new Error(`Block ${index} not found in chain `)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
throw new Error(`Can't merge block into a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {VhdFooter}
|
||||
*/
|
||||
getMetadata() {
|
||||
return this.#disks[this.#disks.length - 1].getMetadata()
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
throw new Error(`Can't flush metadata on a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
*/
|
||||
mergeMetadata(childDisk) {
|
||||
throw new Error(`Can't merge metadata on a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
return this.#disks[0].isDifferencing()
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
throw new Error(`Can't rename a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes all the disks
|
||||
*/
|
||||
async unlink() {
|
||||
for (const disk of this.#disks) {
|
||||
await disk.unlink()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,14 @@
|
||||
/**
|
||||
*
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
import { DiskChain } from '@xen-orchestra/disk-transform'
|
||||
import { RemoteVhd } from './RemoteVhd.mjs'
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
import { defer } from 'golike-defer'
|
||||
/**
|
||||
* @param {any} $defer
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
@@ -15,15 +17,18 @@ import { defer } from 'golike-defer'
|
||||
*/
|
||||
async function _openDiskChain($defer, { handler, path, until }) {
|
||||
let disk
|
||||
/**
|
||||
* @type {Array<RemoteDisk>}
|
||||
*/
|
||||
const disks = []
|
||||
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
|
||||
await disk.init()
|
||||
disks.push(disk)
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent()
|
||||
if (disk.path === until) {
|
||||
if (disk.getPath() === until) {
|
||||
break
|
||||
}
|
||||
disks.unshift(disk)
|
||||
|
||||
@@ -45,7 +45,7 @@ function formatVmBackup(backup) {
|
||||
name_label: backup.vm.name_label,
|
||||
tags: backup.vm.tags,
|
||||
},
|
||||
|
||||
tags: backup.tags ?? [],
|
||||
differencingVhds,
|
||||
dynamicVhds,
|
||||
withMemory,
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.67.2",
|
||||
"version": "0.68.2",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
},
|
||||
@@ -26,13 +26,13 @@
|
||||
"@vates/disposable": "^0.1.6",
|
||||
"@vates/fuse-vhd": "^2.1.2",
|
||||
"@vates/generator-toolbox": "^1.1.0",
|
||||
"@vates/nbd-client": "^3.2.2",
|
||||
"@vates/nbd-client": "^3.2.3",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/disk-transform": "^1.2.1",
|
||||
"@xen-orchestra/fs": "^4.6.5",
|
||||
"@xen-orchestra/fs": "^4.6.6",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/qcow2": "^1.1.1",
|
||||
"@xen-orchestra/qcow2": "^1.1.2",
|
||||
"@xen-orchestra/template": "^0.1.0",
|
||||
"app-conf": "^3.0.0",
|
||||
"compare-versions": "^6.0.0",
|
||||
@@ -48,10 +48,10 @@
|
||||
"parse-pairs": "^2.0.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"tar": "^6.1.15",
|
||||
"tar": "^7.5.3",
|
||||
"uuid": "^9.0.0",
|
||||
"value-matcher": "^0.2.0",
|
||||
"vhd-lib": "^4.14.6",
|
||||
"vhd-lib": "^4.14.7",
|
||||
"xen-api": "^4.7.6",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
@@ -62,7 +62,7 @@
|
||||
"tmp": "^0.2.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^8.6.3"
|
||||
"@xen-orchestra/xapi": "^8.6.6"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"node": ">=20.17"
|
||||
},
|
||||
"dependencies": {
|
||||
"vhd-lib": "^4.14.6",
|
||||
"vhd-lib": "^4.14.7",
|
||||
"@vates/generator-toolbox": "^1.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,9 +18,12 @@ export interface FileAccessor {
|
||||
buffer: Buffer | DataView,
|
||||
position: number
|
||||
): Promise<{ bytesRead: number; buffer: Buffer | DataView }>
|
||||
readFile(path: string): Promise<string>
|
||||
writeFile(path: string, content: string, opts?: unknown): Promise<void>
|
||||
getSize(path: string): Promise<number>
|
||||
mktree(path: string): Promise<void>
|
||||
rmtree(path: string): Promise<void>
|
||||
unlink(path: string): Promise<void>
|
||||
outputStream(stream: Readable): Promise<void>
|
||||
rename(path: string, newPath: string): Promise<void>
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/fs",
|
||||
"version": "4.6.5",
|
||||
"version": "4.6.6",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "The File System for Xen Orchestra backups.",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
|
||||
|
||||
@@ -13,8 +13,7 @@ import { synchronized } from 'decorator-synchronized'
|
||||
import { withTimeout } from './utils'
|
||||
import { basename, dirname, normalize as normalizePath } from './path'
|
||||
import { createChecksumStream, validChecksumOfReadStream } from './checksum'
|
||||
import { UNENCRYPTED_ALGORITHM, _getEncryptor } from './_encryptor'
|
||||
import { FsEncryptor } from './encryptors/FsEncryptor'
|
||||
import { DEFAULT_ENCRYPTION_ALGORITHM, UNENCRYPTED_ALGORITHM, _getEncryptor } from './_encryptor'
|
||||
|
||||
const { info, warn } = createLogger('xo:fs:abstract')
|
||||
|
||||
@@ -27,6 +26,9 @@ const computeRate = (hrtime, size) => {
|
||||
const DEFAULT_TIMEOUT = 12e5 // 20 min
|
||||
const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
|
||||
|
||||
const ENCRYPTION_DESC_FILENAME = 'encryption.json'
|
||||
const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
|
||||
|
||||
const WITH_LIMIT = [
|
||||
'closeFile',
|
||||
'copy',
|
||||
@@ -278,6 +280,7 @@ export default class RemoteHandlerAbstract {
|
||||
|
||||
/**
|
||||
* write a stream to a file using a temporary file
|
||||
* on immutable remote it won't use a temp file since it won't be able to delete /rename it
|
||||
*
|
||||
* @param {string} path
|
||||
* @param {ReadableStream} input
|
||||
@@ -370,7 +373,7 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async outputFile(file, data, { dirMode, flags = 'wx' } = {}) {
|
||||
const encryptedData = await this.#encryptor.encryptBuffer(data)
|
||||
const encryptedData = this.#encryptor.encryptData(data)
|
||||
await this._outputFile(normalizePath(file), encryptedData, { dirMode, flags })
|
||||
}
|
||||
|
||||
@@ -381,7 +384,7 @@ export default class RemoteHandlerAbstract {
|
||||
|
||||
async __readFile(file, { flags = 'r' } = {}) {
|
||||
const data = await this._readFile(normalizePath(file), { flags })
|
||||
return this.#encryptor.decryptBuffer(data)
|
||||
return this.#encryptor.decryptData(data)
|
||||
}
|
||||
|
||||
async #rename(oldPath, newPath, { checksum }, createTree = true) {
|
||||
@@ -432,15 +435,75 @@ export default class RemoteHandlerAbstract {
|
||||
async sync() {
|
||||
await this._sync()
|
||||
try {
|
||||
const encryptor = new FsEncryptor(this)
|
||||
await encryptor.init()
|
||||
this.#rawEncryptor = encryptor
|
||||
console.log('UDPATE', encryptor)
|
||||
await encryptor.updateEncryptionMetadata()
|
||||
await this.#checkMetadata()
|
||||
} catch (error) {
|
||||
await this._forget()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #canWriteMetadata() {
|
||||
const list = await this.__list('/', {
|
||||
filter: e => !e.startsWith('.') && e !== ENCRYPTION_DESC_FILENAME && e !== ENCRYPTION_METADATA_FILENAME,
|
||||
})
|
||||
return list.length === 0
|
||||
}
|
||||
|
||||
async #createMetadata() {
|
||||
const encryptionAlgorithm = this._remote.encryptionKey === undefined ? 'none' : DEFAULT_ENCRYPTION_ALGORITHM
|
||||
this.#rawEncryptor = _getEncryptor(encryptionAlgorithm, this._remote.encryptionKey)
|
||||
|
||||
await Promise.all([
|
||||
this._writeFile(normalizePath(ENCRYPTION_DESC_FILENAME), JSON.stringify({ algorithm: encryptionAlgorithm }), {
|
||||
flags: 'w',
|
||||
}), // not encrypted
|
||||
this.__writeFile(ENCRYPTION_METADATA_FILENAME, `{"random":"${randomUUID()}"}`, { flags: 'w' }), // encrypted
|
||||
])
|
||||
}
|
||||
|
||||
async #checkMetadata() {
|
||||
let encryptionAlgorithm = 'none'
|
||||
let data
|
||||
try {
|
||||
// this file is not encrypted
|
||||
data = await this._readFile(normalizePath(ENCRYPTION_DESC_FILENAME))
|
||||
const json = JSON.parse(data)
|
||||
encryptionAlgorithm = json.algorithm
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
encryptionAlgorithm = this._remote.encryptionKey === undefined ? 'none' : DEFAULT_ENCRYPTION_ALGORITHM
|
||||
}
|
||||
|
||||
try {
|
||||
this.#rawEncryptor = _getEncryptor(encryptionAlgorithm, this._remote.encryptionKey)
|
||||
// this file is encrypted
|
||||
const data = await this.__readFile(ENCRYPTION_METADATA_FILENAME)
|
||||
JSON.parse(data)
|
||||
} catch (error) {
|
||||
// can be enoent, bad algorithm, or broken json ( bad key or algorithm)
|
||||
if (encryptionAlgorithm !== 'none') {
|
||||
if (await this.#canWriteMetadata()) {
|
||||
// any other error , but on empty remote => update with remote settings
|
||||
|
||||
info('will update metadata of this remote')
|
||||
return this.#createMetadata()
|
||||
} else {
|
||||
// to add a new encrypted fs remote, the remote directory must be empty; otherwise, metadata.json is not created
|
||||
if (error.code === 'ENOENT' && error.path.includes('metadata.json')) {
|
||||
throw new Error('Remote directory must be empty.')
|
||||
}
|
||||
|
||||
warn(
|
||||
`The encryptionKey settings of this remote does not match the key used to create it. You won't be able to read any data from this remote`,
|
||||
{ error }
|
||||
)
|
||||
// will probably send a ERR_OSSL_EVP_BAD_DECRYPT if key is incorrect
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async test() {
|
||||
@@ -499,7 +562,7 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async __writeFile(file, data, { flags = 'wx' } = {}) {
|
||||
const encryptedData = this.#encryptor.encryptBuffer(data)
|
||||
const encryptedData = this.#encryptor.encryptData(data)
|
||||
await this._writeFile(normalizePath(file), encryptedData, { flags })
|
||||
}
|
||||
|
||||
@@ -620,7 +683,7 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
async _outputStream(path, input, { dirMode, validator }) {
|
||||
const tmpPath = `${dirname(path)}/.${Date.now()}.${basename(path)}`
|
||||
const tmpPath = this.isImmutable() ? path : `${dirname(path)}/.${Date.now()}.${basename(path)}`
|
||||
const output = await this._createOutputStream(tmpPath, {
|
||||
dirMode,
|
||||
flags: 'wx',
|
||||
@@ -630,8 +693,11 @@ export default class RemoteHandlerAbstract {
|
||||
if (validator !== undefined) {
|
||||
await validator.call(this, tmpPath)
|
||||
}
|
||||
await this.__rename(tmpPath, path)
|
||||
if (!this.isImmutable()) {
|
||||
await this.__rename(tmpPath, path)
|
||||
}
|
||||
} catch (error) {
|
||||
// this may fail on immutable
|
||||
await this.__unlink(tmpPath)
|
||||
throw error
|
||||
}
|
||||
@@ -719,12 +785,16 @@ export default class RemoteHandlerAbstract {
|
||||
}
|
||||
|
||||
get isEncrypted() {
|
||||
return this.#encryptor.algorithm !== 'NULL_ENCRYPTOR'
|
||||
return this.#encryptor.id !== 'NULL_ENCRYPTOR'
|
||||
}
|
||||
|
||||
get encryptionAlgorithm() {
|
||||
return this.#encryptor?.algorithm ?? UNENCRYPTED_ALGORITHM
|
||||
}
|
||||
|
||||
isImmutable() {
|
||||
return this._remote.immutable === true
|
||||
}
|
||||
}
|
||||
|
||||
// from implementation methods, which names start with `__`, create public
|
||||
|
||||
@@ -22,25 +22,32 @@ class TestHandler extends AbstractHandler {
|
||||
}
|
||||
|
||||
async _closeFile() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _getInfo() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _getSize() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _list() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _openFile() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _rename() {
|
||||
Promise(() => {})
|
||||
}
|
||||
async _rmdir() {
|
||||
Promise(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
const clock = sinon.useFakeTimers()
|
||||
/*
|
||||
|
||||
describe('closeFile()', () => {
|
||||
it(`throws in case of timeout`, async () => {
|
||||
const testHandler = new TestHandler()
|
||||
@@ -110,7 +117,7 @@ describe('rmdir()', () => {
|
||||
await assert.rejects(promise, new TimeoutError())
|
||||
})
|
||||
})
|
||||
*/
|
||||
|
||||
describe('encryption', () => {
|
||||
let dir
|
||||
beforeEach(async () => {
|
||||
@@ -140,22 +147,19 @@ describe('encryption', () => {
|
||||
assert.rejects(async () => JSON.parse(await fs.readFile(`${dir}/metadata.json`)))
|
||||
})
|
||||
|
||||
it('sync should not modify existing metadata if check fails', async () => {
|
||||
it('sync should not modify existing metadata', async () => {
|
||||
await fs.writeFile(`${dir}/encryption.json`, `{"algorithm": "none"}`)
|
||||
await fs.writeFile(`${dir}/metadata.json`, `{"random": "NOTSORANDOM"}`)
|
||||
|
||||
await Disposable.use(await getSyncedHandler({ url: `file://${dir}` }), noop)
|
||||
|
||||
await assert.rejects(()=> getSyncedHandler({ url: `file://${dir}?encryptionKey="73c1838d7d8a6088ca2317fb5f29cd00` }))
|
||||
const encryption = JSON.parse(await fs.readFile(`${dir}/encryption.json`, 'utf-8'))
|
||||
assert.equal(encryption.algorithm, 'none')
|
||||
const metadata = JSON.parse(await fs.readFile(`${dir}/metadata.json`, 'utf-8'))
|
||||
assert.equal(metadata.random, 'NOTSORANDOM')
|
||||
})
|
||||
|
||||
|
||||
await assert.doesNotReject(()=> getSyncedHandler({ url: `file://${dir}` }))
|
||||
})
|
||||
|
||||
it('should modify metadata if empty and check fails', async () => {
|
||||
it('should modify metadata if empty', async () => {
|
||||
await Disposable.use(getSyncedHandler({ url: `file://${dir}` }), noop)
|
||||
// nothing created without encryption
|
||||
|
||||
@@ -170,7 +174,6 @@ describe('encryption', () => {
|
||||
encryption = JSON.parse(await fs.readFile(`${dir}/encryption.json`, 'utf-8'))
|
||||
assert.equal(encryption.algorithm, 'none')
|
||||
})
|
||||
/*
|
||||
|
||||
it(
|
||||
'sync should work with encrypted',
|
||||
@@ -221,5 +224,5 @@ describe('encryption', () => {
|
||||
await assert.rejects(
|
||||
Disposable.use(getSyncedHandler({ url: `file://${dir}?encryptionKey="73c1838d7d8a6088ca2317fb5f29cd91"` }), noop)
|
||||
)
|
||||
})*/
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* @typedef {import('node:stream').Readable} Readable
|
||||
*/
|
||||
import { randomUUID } from 'crypto'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
export const CHACHA20 = 'chacha20-poly1305'
|
||||
export const AES256 = 'aes-256-gcm'
|
||||
export const DEFAULT_ENCRYPTION_ALGORITHM = CHACHA20
|
||||
export const UNENCRYPTED_ALGORITHM = 'none'
|
||||
export const SUPPORTED_ALGORITHM = new Set([UNENCRYPTED_ALGORITHM, AES256, CHACHA20, DEFAULT_ENCRYPTION_ALGORITHM])
|
||||
|
||||
export const LEGACY_UPDATE_MODE = 'single'
|
||||
export const ENCRYPTION_DESC_FILENAME = '/encryption.json'
|
||||
export const ENCRYPTION_METADATA_FILENAME = '/metadata.json'
|
||||
|
||||
const { warn } = createLogger('xo:fs:encryptor:abstract')
|
||||
export class AbstractEncryptor {
|
||||
#handler
|
||||
|
||||
constructor(handler) {
|
||||
this.#handler = handler
|
||||
}
|
||||
/**
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init() {
|
||||
|
||||
}
|
||||
/**
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
encryptStream(stream) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
/**
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
decryptStream(stream) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
encryptBuffer(buffer) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
decryptBuffer(buffer) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
/**
|
||||
*update the key and algorithm used
|
||||
* @param {Buffer} key
|
||||
* @param {string} algorithm
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async updateEncryptionKey(key, algorithm) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} algorithm
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async updateEncryptionMetadata( algorithm) {
|
||||
if(algorithm === 'none'){
|
||||
await Promise.all([
|
||||
// this one is not encrypted,but only contains the algorithm
|
||||
this.#handler.unlink(ENCRYPTION_DESC_FILENAME),
|
||||
// this one is encrypted and used as an encryption test
|
||||
this.#handler.unlink(ENCRYPTION_METADATA_FILENAME)
|
||||
])
|
||||
} else {
|
||||
await Promise.all([
|
||||
// this one is not encrypted,but only contains the algorithm
|
||||
this.#handler._writeFile(ENCRYPTION_DESC_FILENAME, JSON.stringify({ algorithm }), {
|
||||
flags: 'w',
|
||||
}),
|
||||
// this one is encrypted and used as an encryption test
|
||||
this.#handler.__writeFile(ENCRYPTION_METADATA_FILENAME, `{"random":"${randomUUID()}"}`, { flags: 'w' }),
|
||||
])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async check() {
|
||||
try {
|
||||
const encryptedRawData = await this.#handler._readFile(ENCRYPTION_METADATA_FILENAME)
|
||||
const decrypted = this.decryptBuffer(encryptedRawData).toString()
|
||||
JSON.parse(decrypted)
|
||||
} catch (error) {
|
||||
if(error.code ==='ENOENT'){
|
||||
// an empty remote that has just been created
|
||||
return
|
||||
}
|
||||
console.log('ERROR', error.code)
|
||||
warn(
|
||||
`The encryptionKey settings of this remote does not match the key used to create it. You won't be able to read any data from this remote`,
|
||||
{ error } )
|
||||
// will probably send a ERR_OSSL_EVP_BAD_DECRYPT if key is incorrect
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns {Promise<{algorithm:string, updateMode: string}>}
|
||||
*/
|
||||
async getEncryptionMetadata() {
|
||||
try {
|
||||
// read file unencrypted
|
||||
const data = await this.#handler._readFile(ENCRYPTION_DESC_FILENAME)
|
||||
const json = JSON.parse(data)
|
||||
console.log({json, data})
|
||||
return { algorithm: json.algorithm, updateMode: json.updateMode ?? LEGACY_UPDATE_MODE}
|
||||
} catch (error) {
|
||||
|
||||
if (error.code === 'ENOENT'){
|
||||
if(this.#handler._remote.encryptionKey){
|
||||
return { algorithm: DEFAULT_ENCRYPTION_ALGORITHM, updateMode: LEGACY_UPDATE_MODE }
|
||||
}
|
||||
return { algorithm: 'none', updateMode: LEGACY_UPDATE_MODE }
|
||||
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { suite, test } from 'node:test'
|
||||
import { AbstractEncryptor, ENCRYPTION_DESC_FILENAME, ENCRYPTION_METADATA_FILENAME } from './AbstractEncryptor.js'
|
||||
|
||||
class MockFs {
|
||||
files = {}
|
||||
_remote = {
|
||||
encryptionKey: undefined,
|
||||
}
|
||||
async _readFile(path) {
|
||||
if (!this.files[path]) {
|
||||
const error = new Error('Not here')
|
||||
error.code = 'ENOENT'
|
||||
throw error
|
||||
}
|
||||
return Promise.resolve(Buffer.from(this.files[path]))
|
||||
}
|
||||
async _writeFile(path, buffer) {
|
||||
this.files[path] = buffer
|
||||
return Promise.resolve()
|
||||
}
|
||||
async __writeFile(path, buffer) {
|
||||
this.files[path] = 'encrypted' + buffer
|
||||
return Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
suite('it should handle metadata', async () => {
|
||||
test('it should return none on empty remote ', async () => {
|
||||
const encryptor = new AbstractEncryptor(new MockFs())
|
||||
const { algorithm } = await encryptor.getAlgorithm()
|
||||
assert.strictEqual(algorithm, 'none')
|
||||
})
|
||||
test('it should read the agorithm if set ', async () => {
|
||||
const fs = new MockFs()
|
||||
fs.files[ENCRYPTION_DESC_FILENAME] = JSON.stringify({ algorithm: 'securealgorithm' })
|
||||
const encryptor = new AbstractEncryptor(fs)
|
||||
const { algorithm } = await encryptor.getAlgorithm()
|
||||
assert.strictEqual(algorithm, 'securealgorithm')
|
||||
})
|
||||
test('it should fail if no metadata file on encrypted ', async () => {
|
||||
const fs = new MockFs()
|
||||
fs._remote.encryptionKey = 'mysecurekey'
|
||||
const encryptor = new AbstractEncryptor(fs)
|
||||
await assert.rejects(() => encryptor.getAlgorithm(), { code: 'ENOENT' })
|
||||
})
|
||||
test('it should fail if no metadata file on encrypted ', async () => {
|
||||
const fs = new MockFs()
|
||||
fs._remote.encryptionKey = 'mysecurekey'
|
||||
const encryptor = new AbstractEncryptor(fs)
|
||||
await assert.rejects(() => encryptor.getAlgorithm(), { code: 'ENOENT' })
|
||||
})
|
||||
|
||||
test('it should update the metadata files correctly', async () => {
|
||||
const fs = new MockFs()
|
||||
const encryptor = new AbstractEncryptor(fs)
|
||||
await encryptor.updateEncryptionMetadata('one GOOD key', 'secure algo')
|
||||
let content = JSON.parse(await fs._readFile(ENCRYPTION_DESC_FILENAME))
|
||||
|
||||
assert.strictEqual(content.algorithm, 'secure algo')
|
||||
|
||||
content = await fs._readFile(ENCRYPTION_METADATA_FILENAME)
|
||||
})
|
||||
})
|
||||
@@ -1,102 +0,0 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* @typedef {import('node:stream').Readable} Readable
|
||||
*/
|
||||
import { AbstractEncryptor } from './AbstractEncryptor.js'
|
||||
import { RollingEncryptor } from './RollingEncryptor.js'
|
||||
import { SingleEncryptor } from './SingleEncryptor.js'
|
||||
|
||||
export class FsEncryptor extends AbstractEncryptor {
|
||||
#handler
|
||||
/**
|
||||
* @type {SingleEncryptor| RollingEncryptor|undefined}
|
||||
*/
|
||||
#encryptor
|
||||
get encryptor(){
|
||||
if(this.#encryptor === undefined){
|
||||
throw new Error("Can't access encryptor before init")
|
||||
}
|
||||
return this.#encryptor
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {*} handler
|
||||
*/
|
||||
constructor(handler) {
|
||||
super(handler)
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
async init() {
|
||||
await super.init()
|
||||
const { algorithm, updateMode } = await this.getEncryptionMetadata()
|
||||
console.log('received', { algorithm, updateMode })
|
||||
let encryptor
|
||||
switch (updateMode) {
|
||||
case 'rolling': {
|
||||
const currentEncryptor = new SingleEncryptor(this.#handler, algorithm)
|
||||
encryptor = new RollingEncryptor(currentEncryptor)
|
||||
break
|
||||
}
|
||||
case 'single':
|
||||
encryptor = new SingleEncryptor(this.#handler, algorithm)
|
||||
break
|
||||
default:
|
||||
throw new Error(`remote encryption mode ${updateMode} is not supported`)
|
||||
}
|
||||
await encryptor.init()
|
||||
// check if
|
||||
await encryptor.check()
|
||||
console.log(('CHECK OK ', { algorithm, updateMode }))
|
||||
this.#encryptor = encryptor
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} key
|
||||
* @param {string} algorithm
|
||||
* @returns
|
||||
*/
|
||||
|
||||
updateEncryptionKey(key, algorithm) {
|
||||
return this.encryptor.updateEncryptionKey(key, algorithm)
|
||||
}
|
||||
|
||||
|
||||
updateEncryptionMetadata() {
|
||||
return this.encryptor.updateEncryptionMetadata()
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
encryptBuffer(buffer) {
|
||||
return this.encryptor.encryptBuffer(buffer)
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
decryptBuffer(buffer) {
|
||||
return this.encryptor.decryptBuffer(buffer)
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
encryptStream(stream) {
|
||||
return this.encryptor.encryptStream(stream)
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
decryptStream(stream) {
|
||||
return this.encryptor.decryptStream(stream)
|
||||
}
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
// @ts-check
|
||||
import assert from 'node:assert'
|
||||
import path from 'node:path'
|
||||
import { readChunkStrict } from '@vates/read-chunk'
|
||||
import { AbstractEncryptor, DEFAULT_ENCRYPTION_ALGORITHM } from './AbstractEncryptor.js'
|
||||
import { SingleEncryptor } from './SingleEncryptor.js'
|
||||
import { Readable } from 'node:stream'
|
||||
|
||||
const ENCRYPTOR_METADATA_BASEPATH = 'encryptors'
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {any} source
|
||||
* @param {number} length
|
||||
* @returns {string}
|
||||
*/
|
||||
function pad(source, length) {
|
||||
let res = String(source)
|
||||
while (res.length < length) {
|
||||
res = '0' + res
|
||||
}
|
||||
return res
|
||||
}
|
||||
export class RollingEncryptor extends AbstractEncryptor {
|
||||
get handler() {
|
||||
return this.#currentEncryptor.handler
|
||||
}
|
||||
/**
|
||||
* @type {Map<number, SingleEncryptor}
|
||||
*/
|
||||
#encryptorCache = new Map()
|
||||
/**
|
||||
* @type {number|undefined}
|
||||
*/
|
||||
#currentEncryptorKey
|
||||
|
||||
#currentEncryptor
|
||||
|
||||
/**
|
||||
* the current encryptor is the key to unlock recursively the full chain
|
||||
* @param handler
|
||||
* @param {SingleEncryptor} currentEncryptor
|
||||
*/
|
||||
constructor(currentEncryptor) {
|
||||
super(currentEncryptor.handler)
|
||||
this.#currentEncryptor = currentEncryptor
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init() {
|
||||
await super.init()
|
||||
const encryptorsList = (await this.handler.list(ENCRYPTOR_METADATA_BASEPATH))
|
||||
.filter(name => name.match(/[0-9]{4}/))
|
||||
this.#currentEncryptorKey = encryptorsList.length + 1
|
||||
this.#encryptorCache.set(this.#currentEncryptorKey, this.#currentEncryptor)
|
||||
await this.#instantiateEncryptor(1) // this will try to load the full chain of key
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} encryptorIndex
|
||||
* @returns {Promise<SingleEncryptor>}
|
||||
*/
|
||||
async #instantiateEncryptor(encryptorIndex) {
|
||||
assert.notEqual(this.#currentEncryptorKey, undefined, `Encryptor index must have been init`)
|
||||
assert.ok(encryptorIndex > 0, `Encryptor index must be over zero`)
|
||||
|
||||
const encryptorInCache = this.#encryptorCache.get(encryptorIndex)
|
||||
if(encryptorInCache !== undefined){
|
||||
return Promise.resolve(encryptorInCache)
|
||||
}
|
||||
// ensure we don't have an infinite recursion
|
||||
assert.ok(encryptorIndex < this.#currentEncryptorKey, `Can't reinstantiate current encryptor ${encryptorIndex}, ${this.#currentEncryptorKey} `)
|
||||
|
||||
|
||||
// past encryptors are store one dis
|
||||
// encrypted with the next more recent encryptor
|
||||
try {
|
||||
const indexStr = String(encryptorIndex).padStart(4, '0')
|
||||
const encryptedByIndex = encryptorIndex+1
|
||||
const encryptedBy = await this.#instantiateEncryptor(encryptedByIndex)
|
||||
|
||||
// read raw data
|
||||
const data = await this.handler._readFile(path.join(ENCRYPTOR_METADATA_BASEPATH, indexStr))
|
||||
const decrypted = encryptedBy.decryptBuffer(data).toString()
|
||||
const { algorithm, key } = JSON.parse(decrypted)
|
||||
const encryptor = new SingleEncryptor(this.handler, algorithm, key)
|
||||
return encryptor
|
||||
} catch (error) {
|
||||
const visibleError = new Error(`Can't instantiate key ${encryptorIndex}`)
|
||||
visibleError.cause = error
|
||||
throw visibleError
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} encryptorIndex
|
||||
* @returns {SingleEncryptor}
|
||||
*/
|
||||
#getEncryptor(encryptorIndex) {
|
||||
let encryptor = this.#encryptorCache.get(encryptorIndex)
|
||||
if (encryptor === undefined) {
|
||||
throw new Error(`Encryptor index ${encryptorIndex} doe snot exists `)
|
||||
}
|
||||
|
||||
return encryptor
|
||||
}
|
||||
|
||||
/**
|
||||
* use the current encrypt to encrypt a buffer
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
encryptBuffer(buffer) {
|
||||
if (typeof buffer === 'string') {
|
||||
buffer = Buffer.from(buffer)
|
||||
}
|
||||
const encryptor = this.#currentEncryptor
|
||||
const keyBuffer = Buffer.alloc(4)
|
||||
keyBuffer.writeUInt32BE(this.#currentEncryptorKey)
|
||||
return Buffer.concat([
|
||||
keyBuffer,
|
||||
encryptor.encryptBuffer(
|
||||
Buffer.concat([
|
||||
keyBuffer, // add keybuffer to encrypted data to ensure it is authenticated
|
||||
buffer,
|
||||
])
|
||||
),
|
||||
])
|
||||
}
|
||||
/**
|
||||
* decrypt a buffer with the right encryptor
|
||||
*
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
decryptBuffer(buffer) {
|
||||
if (buffer.length < 4) {
|
||||
throw new Error(`Can't decrypt a buffer so small, size:${buffer.length}`)
|
||||
}
|
||||
const encryptorIndex = buffer.readUInt32BE(0)
|
||||
const encryptor = this.#getEncryptor(encryptorIndex)
|
||||
const decrypted = encryptor.decryptBuffer(buffer.slice(4))
|
||||
|
||||
// check that the encryptor index has not been tampered with
|
||||
const authenticatedEncryptorIndex = decrypted.readUInt32BE(0)
|
||||
assert.strictEqual(authenticatedEncryptorIndex, encryptorIndex)
|
||||
return decrypted.slice(4)
|
||||
}
|
||||
|
||||
/**
|
||||
* use the current encrypt to encrypt a stream
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
encryptStream(stream) {
|
||||
const keyBuffer = Buffer.alloc(4)
|
||||
keyBuffer.writeUInt32BE(this.#currentEncryptorKey)
|
||||
|
||||
const encryptor = this.currentEncryptor
|
||||
|
||||
async function* generator() {
|
||||
yield keyBuffer
|
||||
for await (const data of encryptor.encryptStream(stream)) {
|
||||
yield data
|
||||
}
|
||||
}
|
||||
|
||||
return Readable.from(generator(), { objectMode: false, highWaterMark: 10 * 1024 * 1024 })
|
||||
}
|
||||
|
||||
/**
|
||||
* decrypt a stream with the right encryptor
|
||||
*
|
||||
* @param {Readable} stream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
async decryptStream(stream) {
|
||||
/**
|
||||
* @type {Buffer}
|
||||
*/
|
||||
const keyBuffer = await readChunkStrict(stream, 4)
|
||||
const encryptorIndex = keyBuffer.readUInt32BE(0)
|
||||
|
||||
const encryptor = this.#getEncryptor(encryptorIndex)
|
||||
const decryptedStream = await encryptor.decryptStream(stream)
|
||||
/**
|
||||
* @type {Buffer}
|
||||
*/
|
||||
const authenticatedKeyBuffer = await readChunkStrict(decryptedStream, 4)
|
||||
const authenticatedEncryptorIndex = authenticatedKeyBuffer.readUInt32BE(0)
|
||||
|
||||
assert.strictEqual(authenticatedEncryptorIndex, encryptorIndex)
|
||||
|
||||
async function* generator() {
|
||||
for await (const data of decryptedStream) {
|
||||
yield data
|
||||
}
|
||||
}
|
||||
return Readable.from(generator(), { objectMode: false, highWaterMark: 10 * 1024 * 1024 })
|
||||
}
|
||||
|
||||
/**
|
||||
* add a new encryptor to the chain
|
||||
* the previous encryptor is saved, encrypted by the new key
|
||||
*
|
||||
* @param {Buffer} key
|
||||
* @param {string} algorithm
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async updateEncryptionKey(key, algorithm = DEFAULT_ENCRYPTION_ALGORITHM) {
|
||||
// can only update the key if the current one is ok
|
||||
await this.check()
|
||||
|
||||
const previousEncryptor = this.#currentEncryptor
|
||||
const encryptor = new SingleEncryptor(this.handler, algorithm, key)
|
||||
|
||||
// same key and algorithm, no need to grow the key chain
|
||||
// compare from encryptors to ensure every key is a buffer
|
||||
if(encryptor.key.equals(previousEncryptor.key) && encryptor.algorithm === previousEncryptor.algorithm){
|
||||
return
|
||||
}
|
||||
|
||||
// write the old encryptor data encrypted with the new encryptor
|
||||
const previousEncryptorIndex = pad(this.#currentEncryptorKey, 4)
|
||||
await this.handler.__writeFile(
|
||||
path.join(ENCRYPTOR_METADATA_BASEPATH, previousEncryptorIndex),
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
algorithm: previousEncryptor.algorithm,
|
||||
key: previousEncryptor.key,
|
||||
updated_at: Date.now(),
|
||||
})
|
||||
)
|
||||
)
|
||||
// set the new encryptor as the current one
|
||||
this.#currentEncryptorKey++
|
||||
this.#encryptorCache.set(this.#currentEncryptorKey, encryptor)
|
||||
this.#currentEncryptor = encryptor
|
||||
|
||||
await this.updateEncryptionMetadata()
|
||||
}
|
||||
|
||||
async updateEncryptionMetadata(){
|
||||
const algorithm = this.#currentEncryptor.algorithm
|
||||
super.updateEncryptionMetadata(algorithm)
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { suite, test } from 'node:test'
|
||||
import { SingleEncryptor } from './SingleEncryptor.js'
|
||||
import { RollingEncryptor } from './RollingEncryptor.js'
|
||||
|
||||
class MockFs {
|
||||
files = {}
|
||||
encryptor
|
||||
_remote = {
|
||||
encryptionKey: undefined,
|
||||
}
|
||||
async _readFile(path) {
|
||||
if (!this.files[path]) {
|
||||
const error = new Error('Not here')
|
||||
error.code = 'ENOENT'
|
||||
throw error
|
||||
}
|
||||
return Promise.resolve(Buffer.from(this.files[path]))
|
||||
}
|
||||
async __readFile(path) {
|
||||
const data = await this._readFile(path)
|
||||
return this.encryptor?.decryptBuffer(data) ?? data
|
||||
}
|
||||
async _writeFile(path, buffer) {
|
||||
this.files[path] = buffer
|
||||
return Promise.resolve()
|
||||
}
|
||||
async __writeFile(path, buffer) {
|
||||
const encrypted = (await this.encryptor?.encryptBuffer(buffer)) ?? buffer
|
||||
return this._writeFile(path, encrypted)
|
||||
}
|
||||
async list(dir) {
|
||||
return Promise.resolve(
|
||||
Object.keys(this.files)
|
||||
.filter(file => file.startsWith(dir))
|
||||
.map(file => file.substring(dir.length))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
suite('it works with one key ', async()=>{
|
||||
test('encryption and decryption', async ()=>{
|
||||
const fs = new MockFs()
|
||||
const encryptor = new SingleEncryptor(fs, 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const rolling = new RollingEncryptor(encryptor)
|
||||
await rolling.init()
|
||||
|
||||
const buffer = Buffer.from(" GOT IT GUY")
|
||||
const encrypted = rolling.encryptBuffer(buffer)
|
||||
const decrypted = rolling.decryptBuffer(encrypted)
|
||||
assert.ok(decrypted.equals(buffer))
|
||||
})
|
||||
})
|
||||
|
||||
suite('it works with updated keys ', async () => {
|
||||
|
||||
test('mock works with buffer', async ()=>{
|
||||
const fs = new MockFs()
|
||||
const encryptor = new SingleEncryptor(fs, 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const rolling = new RollingEncryptor(encryptor)
|
||||
fs.encryptor = rolling
|
||||
await rolling.init()
|
||||
|
||||
const buffer = Buffer.from(" GOT IT GUY")
|
||||
await fs.__writeFile("encrytpion1", buffer)
|
||||
const encrypted = await fs._readFile("encrytpion1")
|
||||
const decrypted = await fs.__readFile("encrytpion1")
|
||||
assert.ok(decrypted.equals(buffer))
|
||||
assert.ok(!decrypted.equals(encrypted))
|
||||
})
|
||||
|
||||
|
||||
test('works with buffer ', async () => {
|
||||
const fs = new MockFs()
|
||||
const encryptor = new SingleEncryptor(fs, 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const rolling = new RollingEncryptor(encryptor)
|
||||
fs.encryptor = rolling
|
||||
await rolling.init()
|
||||
|
||||
const content = Buffer.from(' GOT IT GUY')
|
||||
// encrytped with first key
|
||||
await fs.__writeFile('encrytpion1', content)
|
||||
|
||||
await rolling.updateEncryptionKey('023456789ABCDEF0123456789ABCDEF1', 'chacha20-poly1305')
|
||||
// encrytped with second key and algorithm
|
||||
await fs.__writeFile('encrytpion2', content)
|
||||
|
||||
const encrypted1 = await fs._readFile('encrytpion1')
|
||||
assert.ok(!content.equals(encrypted1))
|
||||
const decrypted1 = await fs.__readFile('encrytpion1')
|
||||
assert.ok(content.equals(decrypted1))
|
||||
|
||||
const encrypted2 = await fs._readFile('encrytpion2')
|
||||
assert.ok(!content.equals(encrypted2))
|
||||
assert.ok(!encrypted1.equals(encrypted2))
|
||||
const decrypted2 = await fs.__readFile('encrytpion2')
|
||||
assert.ok(content.equals(decrypted2))
|
||||
|
||||
assert.equal((await fs.list('encryptors')).length, 1)
|
||||
|
||||
await rolling.updateEncryptionKey('023456789ABCDEF0123456789ABCDEF3', 'chacha20-poly1305')
|
||||
await fs.__writeFile('encrytpion3', content)
|
||||
|
||||
const encrypted3 = await fs._readFile('encrytpion3')
|
||||
assert.ok(!content.equals(encrypted3))
|
||||
assert.ok(!encrypted1.equals(encrypted3))
|
||||
const decrypted3 = await fs.__readFile('encrytpion3')
|
||||
assert.ok(content.equals(decrypted3))
|
||||
|
||||
assert.equal((await fs.list('encryptors')).length, 2)
|
||||
|
||||
|
||||
// ensure key length doesn't grow if we use the same key again
|
||||
await rolling.updateEncryptionMetadata('023456789ABCDEF0123456789ABCDEF3', 'chacha20-poly1305')
|
||||
assert.equal((await fs.list('encryptors')).length, 2)
|
||||
})
|
||||
})
|
||||
@@ -1,222 +0,0 @@
|
||||
// @ts-check
|
||||
/**
|
||||
*
|
||||
* @typedef {import('node:crypto').Cipheriv} Cipheriv
|
||||
* @typedef {import('node:crypto').Decipheriv} Decipheriv
|
||||
*
|
||||
* @typedef {import('node:stream').Readable} Readable
|
||||
*/
|
||||
import { AbstractEncryptor, CHACHA20, DEFAULT_ENCRYPTION_ALGORITHM, ENCRYPTION_DESC_FILENAME, ENCRYPTION_METADATA_FILENAME } from './AbstractEncryptor.js'
|
||||
import assert from 'node:assert'
|
||||
import crypto from 'node:crypto'
|
||||
import { Readable } from 'node:stream'
|
||||
import { readChunkStrict } from '@vates/read-chunk'
|
||||
|
||||
export class SingleEncryptor extends AbstractEncryptor {
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#ivLength = 0
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#authTagLength = 0
|
||||
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#algorithm
|
||||
get algorithm() {
|
||||
return this.#algorithm
|
||||
}
|
||||
/**
|
||||
* @type {Buffer}
|
||||
*/
|
||||
#key
|
||||
get key() {
|
||||
return this.#key
|
||||
}
|
||||
#handler
|
||||
get handler() {
|
||||
return this.#handler
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {*} handler
|
||||
* @param {string} algorithm
|
||||
* @param {Buffer|undefined} key
|
||||
* @returns
|
||||
*/
|
||||
constructor(handler, algorithm, key = undefined) {
|
||||
super(handler)
|
||||
this.#handler = handler
|
||||
this.#key = Buffer.from(key ?? handler._remote.encryptionKey ?? '')
|
||||
if (!this.#key.length) {
|
||||
this.#algorithm = 'none'
|
||||
this.#ivLength = 0
|
||||
return
|
||||
} else {
|
||||
this.#algorithm = algorithm
|
||||
}
|
||||
const info = crypto.getCipherInfo(algorithm, { keyLength: this.#key.length })
|
||||
if (info === undefined) {
|
||||
const error = new Error(
|
||||
`Either the algorithm ${algorithm} is not available, or the key length ${
|
||||
this.#key.length
|
||||
} is incorrect. Supported algorithm are ${crypto.getCiphers()}`
|
||||
)
|
||||
error.code = 'BAD_ALGORITHM'
|
||||
throw error
|
||||
}
|
||||
const { ivLength, mode } = info
|
||||
this.#ivLength = ivLength ?? 0
|
||||
const authTagLength = ['gcm', 'ccm', 'ocb'].includes(mode) || algorithm === CHACHA20 ? 16 : 0
|
||||
this.#authTagLength = authTagLength
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
encryptBuffer(buffer) {
|
||||
if (this.#algorithm === 'none') {
|
||||
return buffer
|
||||
}
|
||||
|
||||
const iv = crypto.randomBytes(this.#ivLength)
|
||||
const cipher = crypto.createCipheriv(this.#algorithm, Buffer.from(this.#key), iv)
|
||||
const encrypted = cipher.update(buffer)
|
||||
return Buffer.concat([
|
||||
iv,
|
||||
encrypted,
|
||||
cipher.final(),
|
||||
this.#authTagLength > 0 ? cipher.getAuthTag() : Buffer.alloc(0),
|
||||
])
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} buffer
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
decryptBuffer(buffer) {
|
||||
if (this.#algorithm === 'none') {
|
||||
return buffer
|
||||
}
|
||||
const iv = buffer.slice(0, this.#ivLength)
|
||||
/**
|
||||
* @type {Decipheriv}
|
||||
*/
|
||||
const decipher = crypto.createDecipheriv(this.#algorithm, Buffer.from(this.#key), iv)
|
||||
let encrypted
|
||||
if (this.#authTagLength > 0) {
|
||||
const authTag = buffer.slice(buffer.length - this.#authTagLength)
|
||||
decipher.setAuthTag(authTag)
|
||||
encrypted = buffer.slice(this.#ivLength, buffer.length - this.#authTagLength)
|
||||
} else {
|
||||
encrypted = buffer.slice(this.#ivLength)
|
||||
}
|
||||
const decrypted = decipher.update(encrypted)
|
||||
return Buffer.concat([decrypted, decipher.final()])
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Readable} input
|
||||
* @returns {Readable}
|
||||
*/
|
||||
encryptStream(input) {
|
||||
if (this.#algorithm === 'none') {
|
||||
return input
|
||||
}
|
||||
const iv = crypto.randomBytes(this.#ivLength)
|
||||
const cipher = crypto.createCipheriv(this.#algorithm, Buffer.from(this.#key), iv)
|
||||
const authTagLength = this.#authTagLength
|
||||
async function* generator() {
|
||||
yield iv
|
||||
for await (const data of input) {
|
||||
const encrypted = cipher.update(data)
|
||||
yield encrypted
|
||||
}
|
||||
yield cipher.final()
|
||||
// must write the auth tag at the end of the encryption stream
|
||||
if (authTagLength > 0) {
|
||||
yield cipher.getAuthTag()
|
||||
}
|
||||
}
|
||||
return Readable.from(generator(), { objectMode: false, highWaterMark: 10 * 1024 * 1024 })
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Readable} encryptedStream
|
||||
* @returns {Readable}
|
||||
*/
|
||||
decryptStream(encryptedStream) {
|
||||
if (this.#algorithm === 'none') {
|
||||
return encryptedStream
|
||||
}
|
||||
const ivLength = this.#ivLength
|
||||
const algorithm = this.#algorithm
|
||||
const key = this.#key
|
||||
const authTagLength = this.#authTagLength
|
||||
async function* generator() {
|
||||
const iv = await readChunkStrict(encryptedStream, ivLength)
|
||||
const cipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv)
|
||||
let authTag = Buffer.alloc(0)
|
||||
for await (const data of encryptedStream) {
|
||||
// auth tag length is stored at the end of the stream
|
||||
// we want to keep it and use at the end to validate the decrypted stream
|
||||
if (data.length >= authTagLength) {
|
||||
// fast path, no buffer concat
|
||||
yield cipher.update(authTag)
|
||||
authTag = data.slice(data.length - authTagLength)
|
||||
yield cipher.update(data.slice(0, data.length - authTagLength))
|
||||
} else {
|
||||
// slower since there is a concat
|
||||
const fullData = Buffer.concat([authTag, data])
|
||||
const fullDataLength = fullData.length
|
||||
if (fullDataLength > authTagLength) {
|
||||
authTag = fullData.slice(fullDataLength - authTagLength)
|
||||
yield cipher.update(fullData.slice(0, fullDataLength - authTagLength))
|
||||
} else {
|
||||
authTag = fullData
|
||||
}
|
||||
}
|
||||
}
|
||||
if (authTagLength > 0) {
|
||||
assert.strictEqual(authTagLength, authTag.length)
|
||||
cipher.setAuthTag(authTag)
|
||||
}
|
||||
yield cipher.final()
|
||||
}
|
||||
return Readable.from(generator(), { objectMode: false, highWaterMark: 10 * 1024 * 1024 })
|
||||
}
|
||||
|
||||
async #isFsEmpty() {
|
||||
const list = await this.#handler.list('/', {
|
||||
filter: e => !e.startsWith('.') && e !== ENCRYPTION_DESC_FILENAME && e !== ENCRYPTION_METADATA_FILENAME,
|
||||
})
|
||||
console.log({list})
|
||||
return list.length === 0
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Buffer} key
|
||||
* @param {string} algorithm
|
||||
*/
|
||||
async updateEncryptionKey(key, algorithm = DEFAULT_ENCRYPTION_ALGORITHM) {
|
||||
if (!(await this.#isFsEmpty())) {
|
||||
throw new Error(`Can't update the encryption of a non empty remote`)
|
||||
}
|
||||
this.#algorithm = algorithm
|
||||
this.#key = key
|
||||
await this.updateEncryptionMetadata()
|
||||
}
|
||||
|
||||
async updateEncryptionMetadata(){
|
||||
super.updateEncryptionMetadata(this.#algorithm)
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import assert from 'node:assert'
|
||||
import { suite, test } from 'node:test'
|
||||
import { Readable } from 'node:stream'
|
||||
import { SingleEncryptor } from './SingleEncryptor'
|
||||
|
||||
class MockFs {
|
||||
files = {}
|
||||
_remote = {
|
||||
encryptionKey: undefined,
|
||||
}
|
||||
async _readFile(path) {
|
||||
if (!this.files[path]) {
|
||||
const error = new Error('Not here')
|
||||
error.code = 'ENOENT'
|
||||
throw error
|
||||
}
|
||||
return Promise.resolve(Buffer.from(this.files[path]))
|
||||
}
|
||||
async _writeFile(path, buffer) {
|
||||
this.files[path] = buffer
|
||||
return Promise.resolve()
|
||||
}
|
||||
async __writeFile(path, buffer) {
|
||||
this.files[path] = 'encrypted' + buffer
|
||||
return Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
suite('no encryption', async () => {
|
||||
test('it should not modify buffer on encrypt ', () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'none')
|
||||
const buffer = Buffer.from('MY SOURCE')
|
||||
const encrypted = encryptor.encryptBuffer(buffer)
|
||||
assert.ok(buffer.equals(encrypted))
|
||||
})
|
||||
test('it should not modify buffer on decrypt ', () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'none')
|
||||
const buffer = Buffer.from('MY SOURCE')
|
||||
const encrypted = encryptor.decryptBuffer(buffer)
|
||||
assert.ok(buffer.equals(encrypted))
|
||||
})
|
||||
})
|
||||
|
||||
suite('it can encrypt', async () => {
|
||||
test('it encrypts and decrypt a buffer', () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const buffer = Buffer.from('MY SOURCE')
|
||||
const encrypted = encryptor.encryptBuffer(buffer)
|
||||
const decrypted = encryptor.decryptBuffer(encrypted)
|
||||
assert.ok(buffer.equals(decrypted))
|
||||
assert.ok(!buffer.equals(encrypted))
|
||||
})
|
||||
|
||||
test('it encrypts and decrypt a stream', async () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
let sourceBuffer = Buffer.alloc(0)
|
||||
function* generator() {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const buffer = Buffer.alloc(1024, i)
|
||||
sourceBuffer = Buffer.concat([sourceBuffer, buffer])
|
||||
yield buffer
|
||||
}
|
||||
}
|
||||
const stream = Readable.from(generator(), { objectMode: false })
|
||||
|
||||
const encrypted = encryptor.encryptStream(stream)
|
||||
let result = Buffer.alloc(0)
|
||||
const decrypted = encryptor.decryptStream(encrypted)
|
||||
|
||||
for await (const buffer of decrypted) {
|
||||
result = Buffer.concat([result, buffer])
|
||||
}
|
||||
})
|
||||
|
||||
test('it authenticate a buffer', () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const buffer = Buffer.allocUnsafe(1024 * 1024)
|
||||
const encrypted = encryptor.encryptBuffer(buffer)
|
||||
encrypted.writeUInt8(2, 7) // modify a byte
|
||||
|
||||
assert.throws(() => encryptor.decryptBuffer(encrypted))
|
||||
})
|
||||
|
||||
|
||||
test('it authenticate a buffer', () => {
|
||||
const encryptor = new SingleEncryptor(new MockFs(), 'aes-256-gcm', '0123456789ABCDEF0123456789ABCDEF')
|
||||
const buffer = Buffer.allocUnsafe(1024 * 1024)
|
||||
const encrypted = encryptor.encryptBuffer(buffer)
|
||||
encrypted.writeUInt8(2, 7) // modify a byte
|
||||
|
||||
assert.throws(() => encryptor.decryptBuffer(encrypted))
|
||||
})
|
||||
})
|
||||
@@ -46,9 +46,9 @@ handlers.forEach(url => {
|
||||
describe(url, () => {
|
||||
let handler
|
||||
|
||||
const testWithFileDescriptor = (path, flags, fn, skip = false) => {
|
||||
it('with path', { skip: skip || skipFsNotInAzure() }, () => fn({ file: path, flags }))
|
||||
it('with file descriptor', { skip: skip || skipFsNotInAzure() }, async () => {
|
||||
const testWithFileDescriptor = (path, flags, fn) => {
|
||||
it('with path', { skip: skipFsNotInAzure() }, () => fn({ file: path, flags }))
|
||||
it('with file descriptor', { skip: skipFsNotInAzure() }, async () => {
|
||||
const file = await handler.openFile(path, flags)
|
||||
try {
|
||||
await fn({ file })
|
||||
@@ -198,20 +198,15 @@ handlers.forEach(url => {
|
||||
const start = random(TEST_DATA_LEN - 1)
|
||||
const size = random(TEST_DATA_LEN - start)
|
||||
|
||||
testWithFileDescriptor(
|
||||
'file',
|
||||
'r',
|
||||
async ({ file }) => {
|
||||
const buffer = Buffer.alloc(size)
|
||||
const result = await handler.read(file, buffer, start)
|
||||
assert.deepEqual(result.buffer, buffer)
|
||||
assert.deepEqual(result, {
|
||||
buffer,
|
||||
bytesRead: Math.min(size, TEST_DATA_LEN - start),
|
||||
})
|
||||
},
|
||||
true
|
||||
)
|
||||
testWithFileDescriptor('file', 'r', async ({ file }) => {
|
||||
const buffer = Buffer.alloc(size)
|
||||
const result = await handler.read(file, buffer, start)
|
||||
assert.deepEqual(result.buffer, buffer)
|
||||
assert.deepEqual(result, {
|
||||
buffer,
|
||||
bytesRead: Math.min(size, TEST_DATA_LEN - start),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('#readFile', () => {
|
||||
@@ -326,21 +321,12 @@ handlers.forEach(url => {
|
||||
})(),
|
||||
},
|
||||
({ offset, expected }, title) => {
|
||||
describe(
|
||||
title,
|
||||
() => {
|
||||
testWithFileDescriptor(
|
||||
'file',
|
||||
'r+',
|
||||
async ({ file }) => {
|
||||
await handler.write(file, PATCH_DATA, offset)
|
||||
assert.deepEqual(await handler.readFile('file'), expected)
|
||||
},
|
||||
true
|
||||
)
|
||||
},
|
||||
true
|
||||
)
|
||||
describe(title, () => {
|
||||
testWithFileDescriptor('file', 'r+', async ({ file }) => {
|
||||
await handler.write(file, PATCH_DATA, offset)
|
||||
assert.deepEqual(await handler.readFile('file'), expected)
|
||||
})
|
||||
})
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
@@ -25,6 +25,7 @@ import copyStreamToBuffer from './_copyStreamToBuffer.js'
|
||||
import guessAwsRegion from './_guessAwsRegion.js'
|
||||
import RemoteHandlerAbstract from './abstract'
|
||||
import { basename, join, split } from './path'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
// endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
|
||||
|
||||
@@ -38,6 +39,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
#bucket
|
||||
#dir
|
||||
#s3
|
||||
#immutable = false
|
||||
|
||||
constructor(remote, _opts) {
|
||||
super(remote, _opts)
|
||||
@@ -139,6 +141,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
}
|
||||
}
|
||||
|
||||
isImmutable() {
|
||||
return super.isImmutable() || this.#immutable
|
||||
}
|
||||
_conditionRetry(error) {
|
||||
return ![401, 403, 404, 405].includes(error?.$metadata?.httpStatusCode) && super._conditionRetry(error)
|
||||
}
|
||||
@@ -388,14 +393,34 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
)
|
||||
|
||||
NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined
|
||||
await this.#s3.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: this.#bucket,
|
||||
Delete: {
|
||||
Objects: result.Contents ?? [],
|
||||
try {
|
||||
await this.#s3.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: this.#bucket,
|
||||
Delete: {
|
||||
Objects: result.Contents ?? [],
|
||||
},
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
warn('Unsupported DeleteObjects, fallback to DeleteObject.', { error, $response: error.$response ?? '' })
|
||||
await asyncEach(
|
||||
result.Contents ?? [],
|
||||
async ({ Key }) => {
|
||||
// _unlink will add the prefix, but Key contains everything
|
||||
// also we don't need to check if we delete a directory, since the list only return files
|
||||
await this.#s3.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.#bucket,
|
||||
Key,
|
||||
})
|
||||
)
|
||||
},
|
||||
})
|
||||
)
|
||||
{
|
||||
concurrency: 16,
|
||||
}
|
||||
)
|
||||
}
|
||||
} while (NextContinuationToken !== undefined)
|
||||
}
|
||||
|
||||
@@ -416,6 +441,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
// will automatically add the contentMD5 header to any upload to S3
|
||||
debug(`Object Lock is enable, enable content md5 header`)
|
||||
this.#s3.middlewareStack.use(getApplyMd5BodyChecksumPlugin(this.#s3.config))
|
||||
this.#immutable = true
|
||||
}
|
||||
} catch (error) {
|
||||
// maybe the account doesn't have enough privilege to query the object lock configuration
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
# how often does the lift immutability script will run to check if
|
||||
# some files need to be made mutable
|
||||
liftEvery = 1h
|
||||
liftEvery = "1h"
|
||||
|
||||
# you can add as many remote as you want, if you change the id ( here : remote1)
|
||||
#[remotes.remote1]
|
||||
|
||||
@@ -44,14 +44,14 @@ List of protected files :
|
||||
|
||||
```js
|
||||
const PATHS = [
|
||||
// xo configuration backupq
|
||||
// xo-config-backups/scheduleId/date/metadata.json
|
||||
'xo-config-backups/*/*/data',
|
||||
'xo-config-backups/*/*/data.json',
|
||||
'xo-config-backups/*/*/metadata.json',
|
||||
// pool backupq
|
||||
'xo-pool-metadata-backups/*/metadata.json',
|
||||
'xo-pool-metadata-backups/*/data',
|
||||
// vm backups , xo-vm-backups/<vmuuid>/
|
||||
// xo-pool-metadata-backups/backupId/scheduleId/date/metadata.json
|
||||
'xo-pool-metadata-backups/*/*/*/metadata.json',
|
||||
'xo-pool-metadata-backups/*/*/*/data',
|
||||
// xo-vm-backups/<vmuuid>/
|
||||
'xo-vm-backups/*/*.json',
|
||||
'xo-vm-backups/*/*.xva',
|
||||
'xo-vm-backups/*/*.xva.checksum',
|
||||
|
||||
@@ -17,19 +17,19 @@
|
||||
"xo-lift-remote-immutability": "./liftProtection.mjs"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"version": "1.0.27",
|
||||
"version": "1.0.29",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.1",
|
||||
"@xen-orchestra/backups": "^0.67.2",
|
||||
"@xen-orchestra/backups": "^0.68.2",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"app-conf": "^3.0.0",
|
||||
"chokidar": "^3.6.0",
|
||||
"execa": "^5.0.0",
|
||||
"ms": "^2.1.3",
|
||||
"vhd-lib": "^4.14.6"
|
||||
"vhd-lib": "^4.14.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"rimraf": "^6.0.1",
|
||||
|
||||
@@ -137,11 +137,13 @@ export async function watchRemote(remoteId, { root, immutabilityDuration, rebuil
|
||||
|
||||
// watch the remote for any new VM metadata json file
|
||||
const PATHS = [
|
||||
// xo-config-backups/scheduleId/date/metadata.json
|
||||
'xo-config-backups/*/*/data',
|
||||
'xo-config-backups/*/*/data.json',
|
||||
'xo-config-backups/*/*/metadata.json',
|
||||
'xo-pool-metadata-backups/*/metadata.json',
|
||||
'xo-pool-metadata-backups/*/data',
|
||||
// xo-pool-metadata-backups/backupId/scheduleId/date/metadata.json
|
||||
'xo-pool-metadata-backups/*/*/*/metadata.json',
|
||||
'xo-pool-metadata-backups/*/*/*/data',
|
||||
// xo-vm-backups/<vmuuid>/
|
||||
'xo-vm-backups/*/*.json',
|
||||
'xo-vm-backups/*/*.xva',
|
||||
|
||||
@@ -2,7 +2,13 @@
|
||||
|
||||
## **next**
|
||||
|
||||
- [Host] Fix host dashboard CPU provisioning calculation [Forum#101359](https://xcp-ng.org/forum/topic/11604/xo-6-dedicated-thread-for-all-your-feedback/84) (PR [#9459](https://github.com/vatesfr/xen-orchestra/pull/9459))
|
||||
|
||||
## **0.18.0** (2026-01-29)
|
||||
|
||||
- [VM/New] Fix wording in "Memory" section (PR [#9309](https://github.com/vatesfr/xen-orchestra/pull/9309))
|
||||
- [TreeView] Scroll to current item in list view (PR [#9332](https://github.com/vatesfr/xen-orchestra/pull/9332))
|
||||
- [VM/New] Added VTPM support (PR [#9389](https://github.com/vatesfr/xen-orchestra/pull/9389))
|
||||
|
||||
## **0.17.0** (2025-11-27)
|
||||
|
||||
@@ -14,7 +20,7 @@
|
||||
|
||||
## **0.15.0** (2025-09-30)
|
||||
|
||||
- [settigns] fix key-value alignement and refactor page (PR [#8981](https://github.com/vatesfr/xen-orchestra/pull/8981))
|
||||
- [Settings] fix key-value alignment and refactor page (PR [#8981](https://github.com/vatesfr/xen-orchestra/pull/8981))
|
||||
- [Settings] Add icon for external links on settings page (PR [#8980](https://github.com/vatesfr/xen-orchestra/pull/8980))
|
||||
- [VM/dashboard] Update QuickInfo card in dashboard to show more information (PR [#8952](https://github.com/vatesfr/xen-orchestra/pull/8952))
|
||||
- [StateHero] Update VtsStateHero component and modify usages in every component (PR [#8910](https://github.com/vatesfr/pull/8910))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@xen-orchestra/lite",
|
||||
"version": "0.17.0",
|
||||
"version": "0.18.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "GIT_HEAD=$(git rev-parse HEAD) vite",
|
||||
@@ -28,13 +28,13 @@
|
||||
"@types/file-saver": "^2.0.7",
|
||||
"@types/lodash-es": "^4.17.12",
|
||||
"@types/node": "^18.19.80",
|
||||
"@vates/types": "^1.16.0",
|
||||
"@vates/types": "^1.19.0",
|
||||
"@vitejs/plugin-vue": "^5.2.1",
|
||||
"@vue/tsconfig": "^0.7.0",
|
||||
"@vueuse/core": "^13.0.0",
|
||||
"@vueuse/math": "^13.0.0",
|
||||
"@vueuse/shared": "^13.0.0",
|
||||
"@xen-orchestra/web-core": "^0.39.1",
|
||||
"@xen-orchestra/web-core": "^0.42.0",
|
||||
"complex-matcher": "^1.0.0",
|
||||
"d3-time-format": "^4.1.0",
|
||||
"decorator-synchronized": "^0.6.0",
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
import StoryMenu from '@/components/component-story/StoryMenu.vue'
|
||||
import InfraPoolList from '@/components/infra/InfraPoolList.vue'
|
||||
import { useNavigationStore } from '@/stores/navigation.store'
|
||||
import { onClickOutside } from '@vueuse/core'
|
||||
import { onClickOutside, tryOnMounted, useTimeoutFn, watchImmediate } from '@vueuse/core'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { ref, watch } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
@@ -42,6 +42,53 @@ watch(
|
||||
},
|
||||
{ immediate: true }
|
||||
)
|
||||
|
||||
const getCurrentNodeId = () => {
|
||||
const name = route.name as string | undefined
|
||||
const params = route.params as Record<string, any>
|
||||
if (name?.startsWith('/pool/[uuid]')) {
|
||||
return `pool:${params.uuid}`
|
||||
}
|
||||
if (name?.startsWith('/host/[uuid]')) {
|
||||
return `host:${params.uuid}`
|
||||
}
|
||||
if (name?.startsWith('/vm/[uuid]')) {
|
||||
return `vm:${params.uuid}`
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
const scrollToCurrent = async () => {
|
||||
const id = getCurrentNodeId()
|
||||
if (!id) {
|
||||
return
|
||||
}
|
||||
|
||||
const container = navElement.value as HTMLElement | undefined
|
||||
const target = container?.querySelector<HTMLElement>(`[data-node-id="${CSS.escape(id)}"]`)
|
||||
const hasChildren = !id.startsWith('vm:')
|
||||
|
||||
if (target) {
|
||||
useTimeoutFn(async () => {
|
||||
if (hasChildren) {
|
||||
target.style.scrollMarginTop = '0.8rem'
|
||||
|
||||
useTimeoutFn(async () => {
|
||||
target.style.scrollMarginTop = ''
|
||||
}, 1000)
|
||||
}
|
||||
target.scrollIntoView({ block: hasChildren ? 'start' : 'center', behavior: 'smooth' })
|
||||
}, 200)
|
||||
} else {
|
||||
useTimeoutFn(async () => {
|
||||
await scrollToCurrent()
|
||||
}, 200)
|
||||
}
|
||||
}
|
||||
|
||||
tryOnMounted(scrollToCurrent)
|
||||
watchImmediate(route, scrollToCurrent)
|
||||
</script>
|
||||
|
||||
<style lang="postcss" scoped>
|
||||
@@ -54,14 +101,4 @@ watch(
|
||||
border-right: 1px solid var(--color-neutral-border);
|
||||
background-color: var(--color-neutral-background-primary);
|
||||
}
|
||||
|
||||
.slide-enter-active,
|
||||
.slide-leave-active {
|
||||
transition: transform 0.3s ease;
|
||||
}
|
||||
|
||||
.slide-enter-from,
|
||||
.slide-leave-to {
|
||||
transform: translateX(-37rem);
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -13,10 +13,10 @@ const props = defineProps<{
|
||||
}>()
|
||||
|
||||
const icons: Record<VM_POWER_STATE, IconName> = {
|
||||
[VM_POWER_STATE.RUNNING]: 'legacy:running',
|
||||
[VM_POWER_STATE.PAUSED]: 'legacy:paused',
|
||||
[VM_POWER_STATE.SUSPENDED]: 'legacy:suspended',
|
||||
[VM_POWER_STATE.HALTED]: 'legacy:halted',
|
||||
[VM_POWER_STATE.RUNNING]: 'status:running-circle',
|
||||
[VM_POWER_STATE.PAUSED]: 'status:paused-circle',
|
||||
[VM_POWER_STATE.SUSPENDED]: 'status:suspended-circle',
|
||||
[VM_POWER_STATE.HALTED]: 'status:halted-circle',
|
||||
}
|
||||
|
||||
const icon = computed(() => icons[props.state])
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<template>
|
||||
<VtsSelect v-if="isSelectWidget(widget)" :id="selectId" accent="brand" />
|
||||
<UiRadioButtonGroup v-else-if="isRadioWidget(widget)" accent="brand">
|
||||
<UiRadioButtonGroup v-else-if="isRadioWidget(widget)" accent="brand" gap="wide">
|
||||
<UiRadioButton
|
||||
v-for="choice in widget.choices"
|
||||
:key="choice.label"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
</template>
|
||||
{{ host.name_label }}
|
||||
<template v-if="isMaster" #status>
|
||||
<VtsIcon v-tooltip="t('master')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-tooltip="t('master')" name="status:primary-circle" size="medium" />
|
||||
</template>
|
||||
</UiHeadBar>
|
||||
</template>
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
</template>
|
||||
|
||||
<script lang="ts" setup>
|
||||
import { ACTIVE_STATES } from '@/libs/utils'
|
||||
import type { XenApiHost } from '@/libs/xen-api/xen-api.types.ts'
|
||||
import { useHostStore } from '@/stores/xen-api/host.store.ts'
|
||||
import { useVmMetricsStore } from '@/stores/xen-api/vm-metrics.store.ts'
|
||||
@@ -30,6 +29,7 @@ import UiCard from '@core/components/ui/card/UiCard.vue'
|
||||
import UiCardNumbers from '@core/components/ui/card-numbers/UiCardNumbers.vue'
|
||||
import UiCardTitle from '@core/components/ui/card-title/UiCardTitle.vue'
|
||||
import { cpuProgressThresholds } from '@core/utils/progress.util.ts'
|
||||
import { VM_POWER_STATE } from '@vates/types'
|
||||
import { and } from '@vueuse/math'
|
||||
import { useArrayReduce } from '@vueuse/shared'
|
||||
import { computed } from 'vue'
|
||||
@@ -51,15 +51,11 @@ const hostVms = computed(() => recordsByHostRef.value.get(host.$ref) ?? [])
|
||||
|
||||
const cpusCount = computed(() => Number(host.cpu_info.cpu_count))
|
||||
|
||||
const vCpusCount = useArrayReduce(
|
||||
hostVms,
|
||||
(total, vm) => {
|
||||
if (ACTIVE_STATES.has(vm.power_state)) {
|
||||
return total + (getVmMetricsByOpaqueRef(vm.metrics)?.VCPUs_number ?? vm.VCPUs_at_startup)
|
||||
}
|
||||
const runningVms = computed(() => hostVms.value.filter(vm => vm.power_state === VM_POWER_STATE.RUNNING))
|
||||
|
||||
return total + vm.VCPUs_at_startup
|
||||
},
|
||||
const vCpusCount = useArrayReduce(
|
||||
runningVms,
|
||||
(total, vm) => total + (getVmMetricsByOpaqueRef(vm.metrics)?.VCPUs_number ?? vm.VCPUs_at_startup),
|
||||
0
|
||||
)
|
||||
</script>
|
||||
|
||||
@@ -12,10 +12,15 @@
|
||||
<VtsQuickInfoRow :label="t('master')">
|
||||
<template #value>
|
||||
<template v-if="isMaster">
|
||||
<VtsIcon v-tooltip="t('master')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-tooltip="t('master')" name="status:primary-circle" size="medium" />
|
||||
{{ t('this-host') }}
|
||||
</template>
|
||||
<UiLink v-else-if="masterHost !== undefined" :to="`/host/${masterHost.uuid}/`" size="medium" icon="fa:server">
|
||||
<UiLink
|
||||
v-else-if="masterHost !== undefined"
|
||||
:to="`/host/${masterHost.uuid}/`"
|
||||
size="medium"
|
||||
icon="object:host"
|
||||
>
|
||||
{{ masterHost.name_label }}
|
||||
</UiLink>
|
||||
</template>
|
||||
@@ -83,8 +88,8 @@ const isReady = logicAnd(isPoolReady, isHostReady, isHostMetricsReady)
|
||||
const isRunning = computed(() => isHostRunning(host))
|
||||
|
||||
const powerState = computed<{ text: string; icon: IconName }>(() => ({
|
||||
text: t(`host:status:${isRunning.value ? 'running' : 'halted'}`),
|
||||
icon: isRunning.value ? 'legacy:running' : 'legacy:halted',
|
||||
text: isRunning.value ? t(`host:status:running`) : t('host:status:halted'),
|
||||
icon: isRunning.value ? 'status:running-circle' : 'status:halted-circle',
|
||||
}))
|
||||
|
||||
const date = computed(() => new Date(parseDateTime(Number(host.other_config.boot_time) * 1000)))
|
||||
|
||||
@@ -49,7 +49,7 @@
|
||||
{{ pif.uuid }}
|
||||
</template>
|
||||
<template #addons>
|
||||
<VtsIcon v-if="pif.management" v-tooltip="t('management')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-if="pif.management" v-tooltip="t('management')" name="status:primary-circle" size="medium" />
|
||||
<VtsCopyButton :value="pif.uuid" />
|
||||
</template>
|
||||
</VtsCardRowKeyValue>
|
||||
@@ -61,14 +61,7 @@
|
||||
<template #value>
|
||||
<div class="network">
|
||||
<!-- TODO Remove the span when the link works and the icon is fixed -->
|
||||
<!--
|
||||
<UiComplexIcon size="medium">
|
||||
<VtsIcon icon="fa:network-wired" accent="current" />
|
||||
<VtsIcon accent="success" :icon="faCircle" :overlay-icon="faCheck" />
|
||||
</UiComplexIcon>
|
||||
<a href="">{{ networkNameLabel }}</a>
|
||||
-->
|
||||
<span v-tooltip class="value text-ellipsis">{{ network?.name_label }}</span>
|
||||
<span v-tooltip class="value">{{ network?.name_label }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<template v-if="network?.name_label" #addons>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="host-pif-table">
|
||||
<UiTitle>
|
||||
{{ t('pifs') }}
|
||||
<template #actions>
|
||||
<template #action>
|
||||
<UiButton
|
||||
v-tooltip="t('coming-soon!')"
|
||||
disabled
|
||||
@@ -137,7 +137,7 @@ function getManagementIcon(pif: XenApiPif) {
|
||||
}
|
||||
|
||||
return {
|
||||
icon: icon('legacy:primary'),
|
||||
icon: icon('status:primary-circle'),
|
||||
tooltip: t('management'),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
</VtsQuickInfoRow>
|
||||
<VtsQuickInfoRow :label="t('pool')">
|
||||
<template v-if="pool !== undefined" #value>
|
||||
<UiLink size="medium" :to="`/pool/${pool.uuid}/`" icon="fa:city">
|
||||
<UiLink size="medium" :to="`/pool/${pool.uuid}/`" icon="object:pool">
|
||||
{{ pool.name_label }}
|
||||
</UiLink>
|
||||
</template>
|
||||
@@ -30,10 +30,10 @@
|
||||
<VtsQuickInfoRow :label="t('master')">
|
||||
<template #value>
|
||||
<template v-if="isMaster">
|
||||
<VtsIcon v-tooltip="t('master')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-tooltip="t('master')" name="status:primary-circle" size="medium" />
|
||||
{{ t('this-host') }}
|
||||
</template>
|
||||
<UiLink v-else-if="masterHost !== undefined" size="medium" :to="`/host/${masterHost.uuid}/`" icon="fa:server">
|
||||
<UiLink v-else-if="masterHost !== undefined" size="medium" :to="`/host/${masterHost.uuid}/`" icon="object:host">
|
||||
{{ masterHost.name_label }}
|
||||
</UiLink>
|
||||
</template>
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
<template>
|
||||
<VtsTreeItem v-if="host !== undefined" :expanded="isExpanded" class="infra-host-item">
|
||||
<UiTreeItemLabel :route="{ name: '/host/[uuid]', params: { uuid: host.uuid } }" icon="fa:server" @toggle="toggle()">
|
||||
<VtsTreeItem v-if="host !== undefined" :expanded="isExpanded" class="infra-host-item" :node-id="`host:${host.uuid}`">
|
||||
<UiTreeItemLabel
|
||||
:route="{ name: '/host/[uuid]', params: { uuid: host.uuid } }"
|
||||
icon="object:host"
|
||||
@toggle="toggle()"
|
||||
>
|
||||
{{ host.name_label || '(Host)' }}
|
||||
<template #addons>
|
||||
<VtsIcon v-if="isPoolMaster" v-tooltip="t('master')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-if="isPoolMaster" v-tooltip="t('master')" name="status:primary-circle" size="medium" />
|
||||
<UiCounter
|
||||
v-if="isReady"
|
||||
v-tooltip="t('running-vm', { count: vmCount })"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<VtsTreeItemError v-if="hasError">
|
||||
{{ t('error-no-data') }}
|
||||
</VtsTreeItemError>
|
||||
<VtsTreeLoadingItem v-else-if="!isReady" icon="fa:server">{{ t('loading-hosts') }}</VtsTreeLoadingItem>
|
||||
<VtsTreeLoadingItem v-else-if="!isReady" icon="object:host">{{ t('loading-hosts') }}</VtsTreeLoadingItem>
|
||||
<InfraHostItem v-for="host in hosts" :key="host.$ref" :host-opaque-ref="host.$ref" />
|
||||
</template>
|
||||
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
<template>
|
||||
<VtsTreeList class="infra-pool-list">
|
||||
<VtsTreeItem :expanded="isExpanded">
|
||||
<VtsTreeItem :expanded="isExpanded" :node-id="`pool:${pool?.uuid}`">
|
||||
<VtsTreeItemError v-if="hasError">
|
||||
{{ t('error-no-data') }}
|
||||
</VtsTreeItemError>
|
||||
<VtsTreeLoadingItem v-else-if="!isReady || pool === undefined" icon="fa:city" />
|
||||
<VtsTreeLoadingItem v-else-if="!isReady || pool === undefined" icon="object:pool" />
|
||||
<UiTreeItemLabel
|
||||
v-else
|
||||
:route="{ name: '/pool/[uuid]', params: { uuid: pool.uuid } }"
|
||||
icon="fa:city"
|
||||
icon="object:pool"
|
||||
@toggle="toggle()"
|
||||
>
|
||||
{{ pool.name_label || '(Pool)' }}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<VtsTreeItem v-if="vm !== undefined" ref="rootElement" expanded class="infra-vm-item">
|
||||
<VtsTreeItem v-if="vm !== undefined" ref="rootElement" expanded class="infra-vm-item" :node-id="`vm:${vm.uuid}`">
|
||||
<UiTreeItemLabel v-if="isVisible" :route="{ name: '/vm/[uuid]', params: { uuid: vm.uuid } }" no-indent>
|
||||
{{ vm.name_label || '(VM)' }}
|
||||
<template #icon>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<template>
|
||||
<VtsTreeItemError v-if="hasError">{{ t('error-no-data') }}</VtsTreeItemError>
|
||||
<template v-else-if="!isReady">
|
||||
<VtsTreeLoadingItem v-for="i in 3" :key="i" icon="fa:display" />
|
||||
<VtsTreeLoadingItem v-for="i in 3" :key="i" icon="object:vm" />
|
||||
</template>
|
||||
<InfraVmItem v-for="vm in vms" :key="vm.$ref" :vm-opaque-ref="vm.$ref" />
|
||||
</template>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<VtsModal accent="danger" icon="fa:server">
|
||||
<VtsModal accent="danger" icon="object:host">
|
||||
<template #title>{{ t('unreachable-hosts') }}</template>
|
||||
<template #content>
|
||||
<div class="description">
|
||||
|
||||
@@ -1,26 +1,17 @@
|
||||
<template>
|
||||
<VtsModal accent="warning" icon="fa:satellite">
|
||||
<VtsDeleteModal>
|
||||
<template #title>
|
||||
<I18nT keypath="confirm-delete" scope="global" tag="div">
|
||||
<span class="n-vms">{{ t('n-vms', { n: count }) }}</span>
|
||||
</I18nT>
|
||||
{{ t('n-vms', { n: count }) }}
|
||||
</template>
|
||||
<template #content>
|
||||
{{ t('please-confirm') }}
|
||||
|
||||
<template #confirm>
|
||||
{{ t(`action:delete-vms`, { n: count }) }}
|
||||
</template>
|
||||
<template #buttons>
|
||||
<VtsModalCancelButton>{{ t('action:go-back') }}</VtsModalCancelButton>
|
||||
<VtsModalConfirmButton>
|
||||
{{ t('action:delete-vms', { n: count }) }}
|
||||
</VtsModalConfirmButton>
|
||||
</template>
|
||||
</VtsModal>
|
||||
</VtsDeleteModal>
|
||||
</template>
|
||||
|
||||
<script lang="ts" setup>
|
||||
import VtsModal from '@core/components/modal/VtsModal.vue'
|
||||
import VtsModalCancelButton from '@core/components/modal/VtsModalCancelButton.vue'
|
||||
import VtsModalConfirmButton from '@core/components/modal/VtsModalConfirmButton.vue'
|
||||
import VtsDeleteModal from '@core/components/modal/VtsDeleteModal.vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
defineProps<{
|
||||
@@ -29,9 +20,3 @@ defineProps<{
|
||||
|
||||
const { t } = useI18n()
|
||||
</script>
|
||||
|
||||
<style lang="postcss" scoped>
|
||||
.n-vms {
|
||||
color: var(--color-warning-item-base);
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<VtsModal accent="info" icon="fa:display">
|
||||
<VtsModal accent="info" icon="object:vm">
|
||||
<template #title>
|
||||
{{ t('action:export-n-vms-manually', { n: labelWithUrl.length }) }}
|
||||
</template>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<VtsModal accent="info" icon="fa:display" @confirm="emit('confirm', compressionType)">
|
||||
<VtsModal accent="info" icon="object:vm" @confirm="emit('confirm', compressionType)">
|
||||
<template #title>
|
||||
{{ t('action:export-n-vms', { n: count }) }}
|
||||
</template>
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<VtsRow v-for="(vdi, index) in vmState.existingVdis" :key="index">
|
||||
<VtsRow v-for="vdi in vmState.existingVdis" :key="vdi.id">
|
||||
<BodyCells :item="{ vdi }" />
|
||||
</VtsRow>
|
||||
<VtsRow v-for="(vdi, index) in vmState.vdis" :key="index">
|
||||
<VtsRow v-for="(vdi, index) in vmState.vdis" :key="vdi.name_label">
|
||||
<BodyCells :item="{ vdi, onRemove: () => emit('remove', index) }" />
|
||||
</VtsRow>
|
||||
<VtsRow>
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
</span>
|
||||
</div>
|
||||
<div v-else>
|
||||
<span>{{ t('host-unknown') }}</span>
|
||||
<span>{{ t('host:status:unknown') }}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td v-tooltip class="typo-body-regular-small text-ellipsis device">{{ pif.device }}</td>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<UiHeadBar icon="fa:city">
|
||||
<UiHeadBar icon="object:pool">
|
||||
{{ name }}
|
||||
<template #actions>
|
||||
<RouterLink :to="{ name: '/vm/new' }">
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="pool-host-internal-networks-table">
|
||||
<UiTitle>
|
||||
{{ t('host-internal-networks') }}
|
||||
<template #actions>
|
||||
<template #action>
|
||||
<UiButton
|
||||
v-tooltip="t('coming-soon!')"
|
||||
disabled
|
||||
|
||||
@@ -38,22 +38,14 @@
|
||||
</template>
|
||||
<template #default>
|
||||
<UiCard class="card-container">
|
||||
<UiCardTitle v-tooltip="{ placement: 'bottom-end' }" class="typo-body-bold text-ellipsis">
|
||||
<UiCardTitle v-tooltip="{ placement: 'bottom-end' }" class="typo-body-bold">
|
||||
{{ network.name_label }}
|
||||
</UiCardTitle>
|
||||
<div class="content">
|
||||
<!-- UUID -->
|
||||
<VtsCardRowKeyValue>
|
||||
<template #key>
|
||||
{{ t('uuid') }}
|
||||
</template>
|
||||
<template #value>{{ network.uuid }}</template>
|
||||
<template #addons>
|
||||
<VtsCopyButton :value="network.uuid" />
|
||||
</template>
|
||||
</VtsCardRowKeyValue>
|
||||
<VtsCodeSnippet :content="network.uuid" copy />
|
||||
<!-- DESCRIPTION -->
|
||||
<VtsCardRowKeyValue>
|
||||
<VtsCardRowKeyValue truncate align-top>
|
||||
<template #key>{{ t('description') }}</template>
|
||||
<template #value>
|
||||
<span class="value">{{ network.name_description }}</span>
|
||||
@@ -129,6 +121,7 @@ import PifRow from '@/components/pif/PifRow.vue'
|
||||
import type { XenApiNetwork } from '@/libs/xen-api/xen-api.types'
|
||||
import { usePifStore } from '@/stores/xen-api/pif.store'
|
||||
import VtsCardRowKeyValue from '@core/components/card/VtsCardRowKeyValue.vue'
|
||||
import VtsCodeSnippet from '@core/components/code-snippet/VtsCodeSnippet.vue'
|
||||
import VtsCopyButton from '@core/components/copy-button/VtsCopyButton.vue'
|
||||
import UiButton from '@core/components/ui/button/UiButton.vue'
|
||||
import UiButtonIcon from '@core/components/ui/button-icon/UiButtonIcon.vue'
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="pool-networks-table">
|
||||
<UiTitle>
|
||||
{{ t('networks') }}
|
||||
<template #actions>
|
||||
<template #action>
|
||||
<UiDropdownButton v-tooltip="t('coming-soon!')" disabled>
|
||||
{{ t('new') }}
|
||||
</UiDropdownButton>
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<template v-else>
|
||||
<VtsQuickInfoRow :label="t('master')">
|
||||
<template #value>
|
||||
<UiLink v-if="masterHost" icon="fa:server" :to="`/host/${masterHost.uuid}/`" size="medium">
|
||||
<UiLink v-if="masterHost" icon="object:host" :to="`/host/${masterHost.uuid}/`" size="medium">
|
||||
{{ masterHost.name_label }}
|
||||
</UiLink>
|
||||
<template v-else>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<UiLink
|
||||
v-if="backupNetwork !== undefined"
|
||||
:to="`/pool/${pool.uuid}/network?id=${backupNetwork.uuid}`"
|
||||
icon="fa:network-wired"
|
||||
icon="object:network"
|
||||
size="medium"
|
||||
>
|
||||
{{ backupNetwork.name_label }}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<VtsQuickInfoRow :label="t('default-storage-repository')">
|
||||
<template #value>
|
||||
<template v-if="defaultSr">
|
||||
<VtsIcon name="fa:database" size="medium" />
|
||||
<VtsIcon name="object:sr" size="medium" />
|
||||
{{ defaultSr.name_label }}
|
||||
</template>
|
||||
<template v-else>
|
||||
@@ -19,7 +19,7 @@
|
||||
<VtsQuickInfoRow :label="t('suspend-storage-repository')">
|
||||
<template #value>
|
||||
<template v-if="suspendSr">
|
||||
<VtsIcon name="fa:database" size="medium" />
|
||||
<VtsIcon name="object:sr" size="medium" />
|
||||
{{ suspendSr.name_label }}
|
||||
</template>
|
||||
<template v-else>
|
||||
@@ -30,7 +30,7 @@
|
||||
<VtsQuickInfoRow :label="t('crash-dump-storage-repository')">
|
||||
<template #value>
|
||||
<template v-if="crashDumpSr">
|
||||
<VtsIcon name="fa:database" size="medium" />
|
||||
<VtsIcon name="object:sr" size="medium" />
|
||||
{{ crashDumpSr.name_label }}
|
||||
</template>
|
||||
<template v-else>
|
||||
@@ -42,7 +42,7 @@
|
||||
<template #value>
|
||||
<ul v-if="haSrs !== undefined && haSrs.length > 0">
|
||||
<li v-for="haSr in haSrs" :key="haSr.uuid">
|
||||
<VtsIcon name="fa:database" size="medium" />
|
||||
<VtsIcon name="object:sr" size="medium" />
|
||||
{{ haSr.name_label }}
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
vmRefs.length > 0 && !isSomeExportable && t(isSingleAction ? 'vm-is-running' : 'no-selected-vm-can-be-exported')
|
||||
"
|
||||
:disabled="isDisabled"
|
||||
icon="fa:display"
|
||||
icon="object:vm"
|
||||
@click="openExportModal"
|
||||
>
|
||||
{{ t('action:export-vm', isSingleAction ? 1 : 2) }}
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
<MenuItem :busy="areVmsBusyToStart" :disabled="!areVmsHalted" icon="fa:play" @click="xenApi.vm.start(vmRefs)">
|
||||
{{ t('action:start') }}
|
||||
</MenuItem>
|
||||
<MenuItem :busy="areVmsBusyToStartOnHost" :disabled="!areVmsHalted" icon="fa:server">
|
||||
<MenuItem :busy="areVmsBusyToStartOnHost" :disabled="!areVmsHalted" icon="object:host">
|
||||
{{ t('action:start-on-host') }}
|
||||
<template #submenu>
|
||||
<MenuItem v-for="host in hosts" :key="host.$ref" icon="fa:server" @click="xenApi.vm.startOn(vmRefs, host.$ref)">
|
||||
<MenuItem v-for="host in hosts" :key="host.$ref" icon="object:host" @click="xenApi.vm.startOn(vmRefs, host.$ref)">
|
||||
<div class="wrapper">
|
||||
{{ host.name_label }}
|
||||
<div>
|
||||
<VtsIcon v-if="host.$ref === pool?.master" name="legacy:primary" size="medium" class="star" />
|
||||
<VtsIcon v-if="host.$ref === pool?.master" name="status:primary-circle" size="medium" class="star" />
|
||||
<PowerStateIcon :state="getHostState(host)" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<TitleBar icon="fa:display">
|
||||
<TitleBar icon="object:vm">
|
||||
{{ name }}
|
||||
<template #actions>
|
||||
<MenuList v-if="vm !== undefined" placement="bottom-end">
|
||||
|
||||
@@ -18,11 +18,10 @@
|
||||
<VtsQuickInfoRow :label="t('host')">
|
||||
<template #value>
|
||||
<span v-if="host" class="host-name">
|
||||
<VtsObjectIcon type="host" :state="hostPowerState" size="medium" />
|
||||
<UiLink :to="`/host/${host.uuid}`" size="medium">
|
||||
<UiLink :to="`/host/${host.uuid}`" size="medium" :icon="`object:host:${hostPowerState}`">
|
||||
{{ host.name_label }}
|
||||
</UiLink>
|
||||
<VtsIcon v-if="isPoolMaster" v-tooltip="t('master')" name="legacy:primary" size="medium" />
|
||||
<VtsIcon v-if="isPoolMaster" v-tooltip="t('master')" name="status:primary-circle" size="medium" />
|
||||
</span>
|
||||
<span v-else>
|
||||
{{ t('none') }}
|
||||
@@ -59,7 +58,6 @@ import { useVmGuestMetricsStore } from '@/stores/xen-api/vm-guest-metrics.store.
|
||||
import { useVmMetricsStore } from '@/stores/xen-api/vm-metrics.store.ts'
|
||||
import { useVmStore } from '@/stores/xen-api/vm.store.ts'
|
||||
import VtsIcon from '@core/components/icon/VtsIcon.vue'
|
||||
import VtsObjectIcon from '@core/components/object-icon/VtsObjectIcon.vue'
|
||||
import VtsQuickInfoCard from '@core/components/quick-info-card/VtsQuickInfoCard.vue'
|
||||
import VtsQuickInfoColumn from '@core/components/quick-info-column/VtsQuickInfoColumn.vue'
|
||||
import VtsQuickInfoRow from '@core/components/quick-info-row/VtsQuickInfoRow.vue'
|
||||
@@ -105,10 +103,10 @@ const hostPowerState = computed(() => {
|
||||
const powerState = useMapper(
|
||||
() => vm.power_state,
|
||||
{
|
||||
[VM_POWER_STATE.RUNNING]: { icon: 'legacy:running', text: t('vm:status:running') },
|
||||
[VM_POWER_STATE.HALTED]: { icon: 'legacy:halted', text: t('vm:status:halted') },
|
||||
[VM_POWER_STATE.PAUSED]: { icon: 'legacy:paused', text: t('vm:status:paused') },
|
||||
[VM_POWER_STATE.SUSPENDED]: { icon: 'legacy:suspended', text: t('vm:status:suspended') },
|
||||
[VM_POWER_STATE.RUNNING]: { icon: 'status:running-circle', text: t('vm:status:running') },
|
||||
[VM_POWER_STATE.HALTED]: { icon: 'status:halted-circle', text: t('vm:status:halted') },
|
||||
[VM_POWER_STATE.PAUSED]: { icon: 'status:paused-circle', text: t('vm:status:paused') },
|
||||
[VM_POWER_STATE.SUSPENDED]: { icon: 'status:suspended-circle', text: t('vm:status:suspended') },
|
||||
},
|
||||
VM_POWER_STATE.RUNNING
|
||||
)
|
||||
|
||||
@@ -41,35 +41,13 @@
|
||||
<UiCardTitle>{{ t('vif') }}</UiCardTitle>
|
||||
<div class="content">
|
||||
<!-- UUID -->
|
||||
<VtsCardRowKeyValue>
|
||||
<template #key>
|
||||
{{ t('uuid') }}
|
||||
</template>
|
||||
<template #value>
|
||||
{{ vif.uuid }}
|
||||
</template>
|
||||
<template #addons>
|
||||
<VtsCopyButton :value="vif.uuid" />
|
||||
</template>
|
||||
</VtsCardRowKeyValue>
|
||||
<VtsCodeSnippet :content="vif.uuid" copy />
|
||||
<!-- NETWORK -->
|
||||
<VtsCardRowKeyValue>
|
||||
<template #key>
|
||||
{{ t('network') }}
|
||||
</template>
|
||||
<template #value>
|
||||
<div class="network">
|
||||
<!-- TODO Remove the span when the link works and the icon is fixed -->
|
||||
<!--
|
||||
<UiComplexIcon size="medium">
|
||||
<VtsIcon icon="fa:network-wired" accent="current" />
|
||||
<VtsIcon accent="success" :icon="faCircle" :overlay-icon="faCheck" />
|
||||
</UiComplexIcon>
|
||||
<a href="">{{ networkNameLabel }}</a>
|
||||
-->
|
||||
<span v-tooltip class="text-ellipsis value">{{ network?.name_label }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<template #value>{{ network?.name_label }}</template>
|
||||
<template v-if="network?.name_label" #addons>
|
||||
<VtsCopyButton :value="network.name_label" />
|
||||
</template>
|
||||
@@ -129,9 +107,7 @@
|
||||
<template #key>
|
||||
<div v-if="index === 0">{{ t('ip-addresses') }}</div>
|
||||
</template>
|
||||
<template #value>
|
||||
<span v-tooltip class="text-ellipsis">{{ ip }}</span>
|
||||
</template>
|
||||
<template #value>{{ ip }}</template>
|
||||
<template #addons>
|
||||
<VtsCopyButton :value="ip" />
|
||||
<UiButtonIcon
|
||||
@@ -177,6 +153,7 @@ import { useNetworkStore } from '@/stores/xen-api/network.store'
|
||||
import { useVmGuestMetricsStore } from '@/stores/xen-api/vm-guest-metrics.store'
|
||||
import { useVmStore } from '@/stores/xen-api/vm.store'
|
||||
import VtsCardRowKeyValue from '@core/components/card/VtsCardRowKeyValue.vue'
|
||||
import VtsCodeSnippet from '@core/components/code-snippet/VtsCodeSnippet.vue'
|
||||
import VtsCopyButton from '@core/components/copy-button/VtsCopyButton.vue'
|
||||
import VtsStatus from '@core/components/status/VtsStatus.vue'
|
||||
import UiButton from '@core/components/ui/button/UiButton.vue'
|
||||
@@ -231,11 +208,6 @@ const status = computed(() => (vif.currently_attached ? 'connected' : 'disconnec
|
||||
gap: 0.8rem;
|
||||
}
|
||||
|
||||
.network {
|
||||
display: flex;
|
||||
gap: 0.8rem;
|
||||
}
|
||||
|
||||
.value:empty::before {
|
||||
content: '-';
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="vm-vifs-table">
|
||||
<UiTitle>
|
||||
{{ t('vifs') }}
|
||||
<template #actions>
|
||||
<template #action>
|
||||
<UiButton
|
||||
v-tooltip="t('coming-soon!')"
|
||||
disabled
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user