mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-18 13:00:52 -06:00
Compare commits
604 Commits
web/fix-qu
...
web-core/r
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abc3018b66 | ||
|
|
423f00fefa | ||
|
|
7456ab9faf | ||
|
|
4d30d43a66 | ||
|
|
d1f3efc57f | ||
|
|
1d881fe8cc | ||
|
|
7a9de0de57 | ||
|
|
699fb196fb | ||
|
|
00e0bee372 | ||
|
|
da045e0de1 | ||
|
|
fdb82ed68a | ||
|
|
112c22e748 | ||
|
|
c540a49643 | ||
|
|
f6484a0695 | ||
|
|
5593f27110 | ||
|
|
81813c7e5b | ||
|
|
0bf0a468b5 | ||
|
|
e78275f046 | ||
|
|
89c7551293 | ||
|
|
ed30291993 | ||
|
|
9765435471 | ||
|
|
e2a4bd2231 | ||
|
|
c6ccd987e2 | ||
|
|
45f810c044 | ||
|
|
fbba05bbde | ||
|
|
9dc52f2e5b | ||
|
|
271d123395 | ||
|
|
88a89cc3a9 | ||
|
|
5f7fab9d34 | ||
|
|
8df8ef0af8 | ||
|
|
6fbb961e7f | ||
|
|
6b1a2afc9c | ||
|
|
3ed8c7b689 | ||
|
|
db9c4815b1 | ||
|
|
3e21edad4e | ||
|
|
1b12aebb47 | ||
|
|
2e009535aa | ||
|
|
5af32683ce | ||
|
|
0fef765ce6 | ||
|
|
4508e01644 | ||
|
|
f60203a927 | ||
|
|
5ebefb0af2 | ||
|
|
fa8844037f | ||
|
|
2098c02e7f | ||
|
|
c4763d5eec | ||
|
|
0b78cab0b3 | ||
|
|
bbf3e1e446 | ||
|
|
d94de6090b | ||
|
|
816b60f11b | ||
|
|
6b36b41d68 | ||
|
|
e9caeafd7a | ||
|
|
06c3933756 | ||
|
|
30ff04d88a | ||
|
|
43d86038b1 | ||
|
|
415207d1ba | ||
|
|
b21e5e8c53 | ||
|
|
aa605639e4 | ||
|
|
8ecaa6d817 | ||
|
|
76655a9e05 | ||
|
|
58b56a0749 | ||
|
|
99d19012ad | ||
|
|
c1e5f2e8d7 | ||
|
|
372a2be21d | ||
|
|
3acd35142c | ||
|
|
28c5e5f138 | ||
|
|
a6c508be46 | ||
|
|
875ceb7cab | ||
|
|
d98dd78168 | ||
|
|
df0c69c023 | ||
|
|
df7ffbb90c | ||
|
|
8553f1cd08 | ||
|
|
41de7528bf | ||
|
|
bfb0a052ef | ||
|
|
5bd04b0736 | ||
|
|
4015987b03 | ||
|
|
25db113349 | ||
|
|
b8637097c2 | ||
|
|
9226c51df7 | ||
|
|
14328ad62a | ||
|
|
ac8dfeb7f5 | ||
|
|
bce3c88915 | ||
|
|
658de40f9e | ||
|
|
b413db14e7 | ||
|
|
e6e5031964 | ||
|
|
484b9a02a3 | ||
|
|
a7829414b0 | ||
|
|
39acc6b3a0 | ||
|
|
465c00b71d | ||
|
|
6814f6b2ef | ||
|
|
cc4cc1796b | ||
|
|
1d81c0b26e | ||
|
|
04a6fb3805 | ||
|
|
1302618f90 | ||
|
|
7a39e45ffb | ||
|
|
4bbfa77ec8 | ||
|
|
57414dafce | ||
|
|
e28e768c72 | ||
|
|
8e5809a675 | ||
|
|
56b17ce7f6 | ||
|
|
27a119f037 | ||
|
|
60ba5070c9 | ||
|
|
0efa078190 | ||
|
|
c3dcb8fd50 | ||
|
|
0286f7786a | ||
|
|
f2b1931408 | ||
|
|
f546849494 | ||
|
|
6ae70a424a | ||
|
|
41751a8599 | ||
|
|
7ea8640ad0 | ||
|
|
2994761178 | ||
|
|
d1736e01f7 | ||
|
|
a2e3343b6e | ||
|
|
e325c6af51 | ||
|
|
598ab85446 | ||
|
|
1591749a23 | ||
|
|
800fd97996 | ||
|
|
45aee36ae7 | ||
|
|
2aff87d6d6 | ||
|
|
449e79de21 | ||
|
|
9d657ac7cc | ||
|
|
f4cc40233c | ||
|
|
2cd5ba0319 | ||
|
|
2444861082 | ||
|
|
7ddff121de | ||
|
|
c1e9b83adc | ||
|
|
18d9b33c87 | ||
|
|
673e729afe | ||
|
|
4e0655dc3d | ||
|
|
a78870b473 | ||
|
|
0bf309a4ad | ||
|
|
5bdd76a955 | ||
|
|
e11530ea7c | ||
|
|
c70f911b5e | ||
|
|
82f047ef9d | ||
|
|
57aec1ab2d | ||
|
|
385fcf3810 | ||
|
|
6699bd1f5d | ||
|
|
844c07e166 | ||
|
|
7676978c78 | ||
|
|
7054fefdbd | ||
|
|
acd84649ed | ||
|
|
7460bf93d1 | ||
|
|
61e4a77cab | ||
|
|
55130311f3 | ||
|
|
f0cbd1bd72 | ||
|
|
561e2349e9 | ||
|
|
7c83ed63f1 | ||
|
|
9525abdee5 | ||
|
|
c6172b7b74 | ||
|
|
1b865bbf89 | ||
|
|
815f4be76c | ||
|
|
53f5c63fa2 | ||
|
|
4ffd1b4e13 | ||
|
|
d25d15efd7 | ||
|
|
322205d203 | ||
|
|
89494457a1 | ||
|
|
6bc2844092 | ||
|
|
53c15bc9e4 | ||
|
|
1dd8c1603f | ||
|
|
52513fc898 | ||
|
|
ed508ec04f | ||
|
|
21823bf430 | ||
|
|
e36e1012e2 | ||
|
|
3fcf582087 | ||
|
|
f1389cff6e | ||
|
|
438a7cc746 | ||
|
|
31e2ff1b02 | ||
|
|
44a23fb33f | ||
|
|
f8501c1a1e | ||
|
|
6cab502260 | ||
|
|
157e7fe757 | ||
|
|
0866a5db81 | ||
|
|
1c245acca5 | ||
|
|
58c2c518a3 | ||
|
|
755ac108f2 | ||
|
|
f88c0e8be2 | ||
|
|
109e37695c | ||
|
|
a4b480de4a | ||
|
|
db8addbc45 | ||
|
|
9dbc090116 | ||
|
|
353678e754 | ||
|
|
a313986694 | ||
|
|
983ac2d2c9 | ||
|
|
4e50858526 | ||
|
|
c497ae3d38 | ||
|
|
9d9b5d5e33 | ||
|
|
5834070e77 | ||
|
|
5c5f27f92b | ||
|
|
8fc04bc03b | ||
|
|
87135c7469 | ||
|
|
fb030a590e | ||
|
|
0be2392519 | ||
|
|
546d42427c | ||
|
|
fc86fda13d | ||
|
|
b44bed6d10 | ||
|
|
4e66889233 | ||
|
|
10bb1741e7 | ||
|
|
a0f890f301 | ||
|
|
839a484017 | ||
|
|
dc9e029bdb | ||
|
|
f203b1f1e1 | ||
|
|
7399a9fec5 | ||
|
|
c69b2336f6 | ||
|
|
80ab29de29 | ||
|
|
1449a8482e | ||
|
|
ec9806e081 | ||
|
|
c6c31524e1 | ||
|
|
49fce9b61f | ||
|
|
f52a35c093 | ||
|
|
9d74585bd4 | ||
|
|
12f4360100 | ||
|
|
771ce8c6d5 | ||
|
|
41c9a6c7c2 | ||
|
|
7e08a016bd | ||
|
|
91c5d98489 | ||
|
|
b895f48a4f | ||
|
|
d68d878747 | ||
|
|
5155a9d5f1 | ||
|
|
7437427fae | ||
|
|
a16b4cb540 | ||
|
|
21508c97bc | ||
|
|
e8ff60c8bc | ||
|
|
153efccb85 | ||
|
|
a99e1aa72d | ||
|
|
dd5f0cdaa9 | ||
|
|
ea142b50a4 | ||
|
|
5b5a0a72cc | ||
|
|
84a16ef85f | ||
|
|
b983e6d4fa | ||
|
|
a0e771f09c | ||
|
|
9bd91aea43 | ||
|
|
0a042d2f54 | ||
|
|
c610951450 | ||
|
|
a831ac7b21 | ||
|
|
d0f71d4ec7 | ||
|
|
c1a9bc82fb | ||
|
|
601cc3e9fd | ||
|
|
79d1f42dd7 | ||
|
|
64c366838a | ||
|
|
85b3162be6 | ||
|
|
eb020e250b | ||
|
|
56142085e5 | ||
|
|
2838bde875 | ||
|
|
fa110ed9c9 | ||
|
|
d3187c06b5 | ||
|
|
2f41ccffe8 | ||
|
|
a9e45fcb8e | ||
|
|
1e69c25ba5 | ||
|
|
9f852365fc | ||
|
|
05824cb8a6 | ||
|
|
638ee77aff | ||
|
|
08a560a8da | ||
|
|
e77a0e43f1 | ||
|
|
8069adf26d | ||
|
|
1241977599 | ||
|
|
4ca314fbc7 | ||
|
|
13ab1cce86 | ||
|
|
9f7797fdda | ||
|
|
21767145bd | ||
|
|
6805e16108 | ||
|
|
ecd49e69a7 | ||
|
|
a5f5a0273e | ||
|
|
ddc9bf3211 | ||
|
|
75db97f9ea | ||
|
|
07c1c02f03 | ||
|
|
dbe8e0ce26 | ||
|
|
82cd861479 | ||
|
|
32f8c1a25f | ||
|
|
9e7813319a | ||
|
|
caba00a2b9 | ||
|
|
927561123a | ||
|
|
097c2bd1d9 | ||
|
|
5288b6d848 | ||
|
|
263059be22 | ||
|
|
c83df8a342 | ||
|
|
4f959e2d22 | ||
|
|
ee6df8c979 | ||
|
|
0a0ae5dc7e | ||
|
|
58b6215c41 | ||
|
|
0e56c5e2bd | ||
|
|
0b44b1b52b | ||
|
|
4c7b8b194f | ||
|
|
ac2ed8880a | ||
|
|
6fc30c899d | ||
|
|
05e1298eec | ||
|
|
d20561fadc | ||
|
|
53a3e2a70f | ||
|
|
2c90eb5ba8 | ||
|
|
c3e78ebdfd | ||
|
|
264b93d502 | ||
|
|
d0abf8810c | ||
|
|
bdd9b26651 | ||
|
|
66d5014755 | ||
|
|
8e2d9f1022 | ||
|
|
566bdeda63 | ||
|
|
e86d71fc31 | ||
|
|
4827f33305 | ||
|
|
9b6d1089f4 | ||
|
|
d1f8f61791 | ||
|
|
dbf00ba0cd | ||
|
|
44a0d14e88 | ||
|
|
0b52a05bb3 | ||
|
|
407591d74d | ||
|
|
0b29586626 | ||
|
|
42eb8664be | ||
|
|
1d8c429532 | ||
|
|
c51df2239f | ||
|
|
b89c26459c | ||
|
|
a1333c07d7 | ||
|
|
5e8e5beba1 | ||
|
|
9169e11f35 | ||
|
|
6908001bd8 | ||
|
|
53ec9a62a8 | ||
|
|
4a0590b7e9 | ||
|
|
d358aed897 | ||
|
|
5fcb6db32e | ||
|
|
1de60c52b4 | ||
|
|
55243b65a2 | ||
|
|
0247b3b9b3 | ||
|
|
6dae628e0d | ||
|
|
7827b27d63 | ||
|
|
86ddff46a9 | ||
|
|
476f4e5f1f | ||
|
|
8448b4d5a6 | ||
|
|
28952a0a4a | ||
|
|
19f973fcd3 | ||
|
|
d9fe9b603f | ||
|
|
13a462429e | ||
|
|
bfe788e096 | ||
|
|
980e4b5887 | ||
|
|
0d8cb990c6 | ||
|
|
7794adfb89 | ||
|
|
fb67e8eea7 | ||
|
|
1316a811f5 | ||
|
|
c6f47f9ef3 | ||
|
|
c2a95467f8 | ||
|
|
4d58ff885a | ||
|
|
3892eee7a0 | ||
|
|
15b9e13ed2 | ||
|
|
e504b8af0f | ||
|
|
efc751c54e | ||
|
|
8b4c953ab0 | ||
|
|
32b3c0b5fc | ||
|
|
7c1764a39e | ||
|
|
b248cddece | ||
|
|
df2dc3a710 | ||
|
|
227063341e | ||
|
|
a4ab2e8b06 | ||
|
|
9f3872a5f1 | ||
|
|
e2d727abe5 | ||
|
|
11720d96f1 | ||
|
|
859ebb892d | ||
|
|
035ee8fb8e | ||
|
|
bc69046bdd | ||
|
|
debde3f16e | ||
|
|
3eb9efe1f2 | ||
|
|
7cf9726876 | ||
|
|
64d9f8c39e | ||
|
|
03cea8b24d | ||
|
|
4b4c9b91f3 | ||
|
|
907ee7d88c | ||
|
|
dbdc4cac4a | ||
|
|
19c0b958fc | ||
|
|
e5d2e83de6 | ||
|
|
85596da792 | ||
|
|
230749a3f3 | ||
|
|
b667bc8a23 | ||
|
|
76abf1ed73 | ||
|
|
54930ee07f | ||
|
|
90295ca924 | ||
|
|
e281c536d3 | ||
|
|
efa5b1a71e | ||
|
|
62aaa21ced | ||
|
|
dfbf654159 | ||
|
|
4bc614ffd1 | ||
|
|
1640a6ae00 | ||
|
|
66b677af28 | ||
|
|
655bda4789 | ||
|
|
07e605162a | ||
|
|
ce98c3deba | ||
|
|
71fa8924d7 | ||
|
|
9c78018662 | ||
|
|
72f31d412d | ||
|
|
ef60450dc8 | ||
|
|
f4988c00af | ||
|
|
43c9f574fd | ||
|
|
a15eeeed5b | ||
|
|
14a72a2612 | ||
|
|
68554b3e0a | ||
|
|
91d5107e1d | ||
|
|
8e46b83d3e | ||
|
|
6105681df2 | ||
|
|
05713d16c0 | ||
|
|
1c6bc86794 | ||
|
|
21b4d1ee34 | ||
|
|
0c92bf559a | ||
|
|
5d28710ab3 | ||
|
|
3126688d5e | ||
|
|
e70fe39934 | ||
|
|
0af02941ed | ||
|
|
779876ede9 | ||
|
|
dd8bb763e5 | ||
|
|
5e6c19b39f | ||
|
|
dbb76cce6e | ||
|
|
5cf1de5356 | ||
|
|
bed27e5ffa | ||
|
|
bff4e4d4e2 | ||
|
|
97d202dae0 | ||
|
|
460bbc4569 | ||
|
|
708089622d | ||
|
|
1818a031af | ||
|
|
5cc6154bd2 | ||
|
|
6cd0710f89 | ||
|
|
65858c9ce7 | ||
|
|
6277a12c4a | ||
|
|
16de2e9fca | ||
|
|
39fa0f041f | ||
|
|
ca3a7540f4 | ||
|
|
0bac1d3e0e | ||
|
|
99f221ccbf | ||
|
|
f24952f967 | ||
|
|
566b210d1e | ||
|
|
5fc2e4e8e5 | ||
|
|
65de89c7ee | ||
|
|
1c55ca61e2 | ||
|
|
ed5bd4d42e | ||
|
|
488eec9d1c | ||
|
|
89d45caac9 | ||
|
|
2592f2fc2c | ||
|
|
87547a2130 | ||
|
|
5349f2ad09 | ||
|
|
d34025fd25 | ||
|
|
c53758ddc5 | ||
|
|
90d1207a3c | ||
|
|
9f478f97a2 | ||
|
|
e86bd71819 | ||
|
|
53c1a33106 | ||
|
|
7c06abc99f | ||
|
|
2a5b0ffd1a | ||
|
|
e494007b2a | ||
|
|
c2fb0470fc | ||
|
|
5fb5e639cd | ||
|
|
da3733a3a9 | ||
|
|
bd7ba64b68 | ||
|
|
436e2ea168 | ||
|
|
dcbd550961 | ||
|
|
8811bb4bac | ||
|
|
2d06677887 | ||
|
|
6cfefc91e4 | ||
|
|
8dc7254128 | ||
|
|
18a41cd055 | ||
|
|
a7ad70bbed | ||
|
|
fa0202f5b3 | ||
|
|
88b88187e9 | ||
|
|
ade825df55 | ||
|
|
042c9c645b | ||
|
|
6bf5f4b7d4 | ||
|
|
063f499ee0 | ||
|
|
e4d484e84f | ||
|
|
05c6f87a12 | ||
|
|
8a390057b6 | ||
|
|
82d65e3c68 | ||
|
|
58f029dcbe | ||
|
|
7112c33142 | ||
|
|
08c2923386 | ||
|
|
ed5be40099 | ||
|
|
8c038bac34 | ||
|
|
961b505cfb | ||
|
|
32556029cd | ||
|
|
65013190e3 | ||
|
|
2dd451a7d9 | ||
|
|
73a8ba59ba | ||
|
|
285d1da361 | ||
|
|
4d77b79ce9 | ||
|
|
268033439b | ||
|
|
df3a365eb7 | ||
|
|
61ee81ea3a | ||
|
|
2789b339cb | ||
|
|
c895f05486 | ||
|
|
ed3f50cb1a | ||
|
|
6bb90082b9 | ||
|
|
c870bb653a | ||
|
|
b11484ce6d | ||
|
|
cfb64f8e0d | ||
|
|
d6e0f70510 | ||
|
|
b47a7055a2 | ||
|
|
a101e9e02d | ||
|
|
e547447bb4 | ||
|
|
5d92f813f6 | ||
|
|
ab4e20b0a5 | ||
|
|
ff4b087be8 | ||
|
|
09208ba0fb | ||
|
|
896e9386a9 | ||
|
|
87471d90c4 | ||
|
|
740fc03a04 | ||
|
|
45ef65b251 | ||
|
|
bf705e0a98 | ||
|
|
989864314f | ||
|
|
0a377f9568 | ||
|
|
cb96de6d11 | ||
|
|
7e032e9f84 | ||
|
|
51855c22d4 | ||
|
|
676b059485 | ||
|
|
81d0d28819 | ||
|
|
9123921676 | ||
|
|
544590a860 | ||
|
|
63e7eb881c | ||
|
|
030d697392 | ||
|
|
af56933587 | ||
|
|
ec0bdb3286 | ||
|
|
70055960d0 | ||
|
|
a9ab882b75 | ||
|
|
bce7408f1f | ||
|
|
c8abbb259e | ||
|
|
35cc755bf0 | ||
|
|
5750b58362 | ||
|
|
8a95941297 | ||
|
|
8289150460 | ||
|
|
87df162944 | ||
|
|
e9eda42d7d | ||
|
|
d942d144ca | ||
|
|
d829816f62 | ||
|
|
8b3b23e995 | ||
|
|
52807ed15f | ||
|
|
596f9ac536 | ||
|
|
d58efc7b2d | ||
|
|
73aa66a219 | ||
|
|
93c8903953 | ||
|
|
dc8c455a0b | ||
|
|
d601558bf4 | ||
|
|
d5cee13e25 | ||
|
|
151d7ed22a | ||
|
|
3d29348337 | ||
|
|
62ab5a2f5b | ||
|
|
c2d1c66a59 | ||
|
|
730d0ff41d | ||
|
|
64d164300b | ||
|
|
04eeb78df7 | ||
|
|
6bdfbfb953 | ||
|
|
e6573e26a9 | ||
|
|
5f5cb9b45d | ||
|
|
940e879a1c | ||
|
|
861567d0db | ||
|
|
8e536a4c32 | ||
|
|
e100736349 | ||
|
|
98e4bcccfa | ||
|
|
938b34343d | ||
|
|
e1ecc2d6d4 | ||
|
|
afabf35c66 | ||
|
|
8e0f48bddd | ||
|
|
70dc82e621 | ||
|
|
2928813613 | ||
|
|
a86d07adea | ||
|
|
a282e10069 | ||
|
|
8a70196fa9 | ||
|
|
1d8ae5eea8 | ||
|
|
055bce7d91 | ||
|
|
1ee07eb06f | ||
|
|
6a5eb12051 | ||
|
|
fbf3f1bc7f | ||
|
|
dedfa91669 | ||
|
|
f3480f6939 | ||
|
|
e48166846c | ||
|
|
f7a79a0be3 | ||
|
|
f0f1fd1715 | ||
|
|
2e4c45b207 | ||
|
|
72a3f0a9da | ||
|
|
2d6f784f0a | ||
|
|
614449d540 | ||
|
|
53ff333778 | ||
|
|
4207954982 | ||
|
|
6af64c7095 | ||
|
|
7d03560a3e | ||
|
|
9f9519ef15 | ||
|
|
0838b112de | ||
|
|
d6e4456538 | ||
|
|
022dc0bf2e | ||
|
|
49909df1d2 | ||
|
|
7c7d7a938e | ||
|
|
a67ad48012 | ||
|
|
6c6d7a377f | ||
|
|
d3006026ca | ||
|
|
62cdd56691 | ||
|
|
775e8a8b12 | ||
|
|
f5676705cc | ||
|
|
0bad96418b | ||
|
|
3222025088 | ||
|
|
86f3096c1b | ||
|
|
d7fe4f1729 | ||
|
|
64a2666645 | ||
|
|
3b7f623f9e | ||
|
|
16cdbc16fa | ||
|
|
52510ed03c | ||
|
|
bbec7d4532 | ||
|
|
cf0449c034 | ||
|
|
9e93ba14ae | ||
|
|
bf330311a1 | ||
|
|
481ef5e585 | ||
|
|
a5debf3241 | ||
|
|
d8c106fc5d | ||
|
|
8d2af27896 | ||
|
|
893cdcbc86 | ||
|
|
241c37b35c | ||
|
|
57e1b94115 |
35
.eslintrc.js
35
.eslintrc.js
@@ -233,7 +233,29 @@ module.exports = {
|
||||
'@intlify/vue-i18n/no-duplicate-keys-in-locale': 'error',
|
||||
'@intlify/vue-i18n/no-missing-keys-in-other-locales': [
|
||||
'error',
|
||||
{ ignoreLocales: ['de', 'fa', 'cs', 'es', 'sv', 'it', 'ru', 'uk', 'nl', 'pt_BR'] },
|
||||
{
|
||||
ignoreLocales: [
|
||||
'de',
|
||||
'fa',
|
||||
'cs',
|
||||
'es',
|
||||
'sv',
|
||||
'it',
|
||||
'ru',
|
||||
'uk',
|
||||
'nl',
|
||||
'pt-BR',
|
||||
'da',
|
||||
'ko',
|
||||
'nb-NO',
|
||||
'ja',
|
||||
'fi',
|
||||
'pl',
|
||||
'zh-Hans',
|
||||
'sk',
|
||||
'pt',
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
@@ -268,6 +290,17 @@ module.exports = {
|
||||
'eslint-comments/no-unlimited-disable': 'off',
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ['*.d.ts'],
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
sourceType: 'module',
|
||||
},
|
||||
rules: {
|
||||
'n/no-unsupported-features/es-syntax': 'off',
|
||||
'no-unused-vars': 'off',
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
ignorePatterns: ['@vates/fatfs/'],
|
||||
|
||||
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@@ -25,7 +25,7 @@ jobs:
|
||||
- name: Setup Node environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '24.14'
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Install project dependencies
|
||||
|
||||
5
@vates/async-each/index.d.ts
vendored
Normal file
5
@vates/async-each/index.d.ts
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
export function asyncEach<Item>(
|
||||
iterable: Iterable<Item> | AsyncIterable<Item>,
|
||||
iteratee: (item: Item, index: number, iterable: Iterable<Item> | AsyncIterable<Item>) => Promise<void>,
|
||||
opts?: { concurrency?: number; signal?: AbortSignal; stopOnError?: boolean }
|
||||
): Promise<void>
|
||||
@@ -73,20 +73,20 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
|
||||
}
|
||||
nextIsRunning = true
|
||||
if (running < concurrency) {
|
||||
const cursor = await it.next()
|
||||
if (cursor.done) {
|
||||
next = () => {
|
||||
if (running === 0) {
|
||||
if (errors.length === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new AggregateError(errors))
|
||||
try {
|
||||
const cursor = await it.next()
|
||||
if (cursor.done) {
|
||||
next = () => {
|
||||
if (running === 0) {
|
||||
if (errors.length === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new AggregateError(errors))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
++running
|
||||
try {
|
||||
} else {
|
||||
++running
|
||||
const result = iteratee.call(this, cursor.value, index++, iterable)
|
||||
let then
|
||||
if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
|
||||
@@ -94,9 +94,9 @@ exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, s
|
||||
} else {
|
||||
onFulfilled(result)
|
||||
}
|
||||
} catch (error) {
|
||||
onRejected(error)
|
||||
}
|
||||
} catch (error) {
|
||||
onRejected(error)
|
||||
}
|
||||
nextIsRunning = false
|
||||
return next()
|
||||
|
||||
@@ -67,20 +67,17 @@ describe('asyncEach', () => {
|
||||
})
|
||||
|
||||
it('stops on first error when stopOnError is true', async () => {
|
||||
const tracker = new assert.CallTracker()
|
||||
|
||||
const error = new Error()
|
||||
const iteratee = tracker.calls((_, i) => {
|
||||
const iteratee = spy((_, i) => {
|
||||
if (i === 1) {
|
||||
throw error
|
||||
}
|
||||
}, 2)
|
||||
})
|
||||
assert.deepStrictEqual(
|
||||
await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true })),
|
||||
error
|
||||
)
|
||||
|
||||
tracker.verify()
|
||||
assert.deepStrictEqual(iteratee.callCount, 2)
|
||||
})
|
||||
|
||||
it('rejects AggregateError when stopOnError is false', async () => {
|
||||
@@ -100,19 +97,30 @@ describe('asyncEach', () => {
|
||||
})
|
||||
|
||||
it('can be interrupted with an AbortSignal', async () => {
|
||||
const tracker = new assert.CallTracker()
|
||||
|
||||
const ac = new AbortController()
|
||||
const iteratee = tracker.calls((_, i) => {
|
||||
const iteratee = spy((_, i) => {
|
||||
if (i === 1) {
|
||||
ac.abort()
|
||||
}
|
||||
}, 2)
|
||||
})
|
||||
await assert.rejects(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal }), {
|
||||
message: 'asyncEach aborted',
|
||||
})
|
||||
assert.deepStrictEqual(iteratee.callCount, 2)
|
||||
})
|
||||
|
||||
tracker.verify()
|
||||
it('handle error on iterable', async () => {
|
||||
const iteratee = spy(async () => {})
|
||||
const iterable = async function* () {
|
||||
for (const value of [1, 2, 3]) {
|
||||
if (value === 2) {
|
||||
throw new Error('iterator error')
|
||||
}
|
||||
await randomDelay()
|
||||
yield value
|
||||
}
|
||||
}
|
||||
await assert.rejects(() => asyncEach(iterable(), iteratee, { concurrency: 1 }), { message: 'iterator error' })
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
@@ -24,7 +24,8 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "1.0.0",
|
||||
"types": "index.d.ts",
|
||||
"version": "1.0.3",
|
||||
"engines": {
|
||||
"node": ">=8.10"
|
||||
},
|
||||
|
||||
@@ -24,8 +24,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/multi-key-map": "^0.2.0",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"ensure-array": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -62,7 +62,7 @@ exports.parseFlags = function (flags) {
|
||||
case 'ax': info = {read:false, write:true, create:true, append:true, exclusive:true}; break;
|
||||
case 'a+': info = {read:true, write:true, create:true, append:true}; break;
|
||||
case 'ax+': info = {read:true, write:true, create:true, append:true, exclusive:true}; break;
|
||||
default: throw Error("Uknown mode: "+flags); // TODO: throw as `S.err.INVAL`
|
||||
default: throw Error("Unknown mode: "+flags); // TODO: throw as `S.err.INVAL`
|
||||
}
|
||||
if (info.sync) throw Error("Mode not implemented."); // TODO: what would this require of us?
|
||||
if (_dir) info._openDir = true;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@vates/fatfs",
|
||||
"version": "0.11.0",
|
||||
"version": "0.11.1",
|
||||
"description": "fs implementation on top of raw FAT16/FAT32 block source",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -90,7 +90,7 @@ function startTests(vol, waitTime) {
|
||||
assert(path === file, "Fixed fluffy path matches normal one.");
|
||||
});
|
||||
fs.realpath([BASE_DIR,"non","existent","path"].join('/'), function (e) {
|
||||
assert(e, "Expected error calling fs.realpath on non-existent file.");
|
||||
assert(e, "Expected error calling fs.realpath on nonexistent file.");
|
||||
});
|
||||
|
||||
fs.readdir(BASE_DIR, function (e, arr) {
|
||||
|
||||
@@ -19,12 +19,12 @@
|
||||
},
|
||||
"main": "./index.mjs",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/fs": "^4.6.3",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"fuse-native": "^2.2.6",
|
||||
"lru-cache": "^7.14.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"vhd-lib": "^4.14.1"
|
||||
"vhd-lib": "^4.16.0"
|
||||
},
|
||||
"bin": {
|
||||
"xo-fuse-vhd": "./cli.mjs"
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "@vates/generator-toolbox",
|
||||
"version": "1.0.4",
|
||||
"version": "1.1.2",
|
||||
"main": "dist/index.mjs",
|
||||
"license": "MIT",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.19.0",
|
||||
"rimraf": "^5.0.1",
|
||||
"rimraf": "^6.0.1",
|
||||
"@types/node": "^20.6",
|
||||
"typescript": "~5.6",
|
||||
"typescript-eslint": "^8.23.0"
|
||||
|
||||
@@ -144,14 +144,7 @@ class Forked<T, TReturn, TNext> implements AsyncGenerator<T, TReturn, TNext> {
|
||||
return this.#parent.remove(this.#uid, e)
|
||||
}
|
||||
|
||||
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
|
||||
while (true) {
|
||||
const res = await this.next()
|
||||
if (res.done) {
|
||||
break
|
||||
}
|
||||
|
||||
yield res.value
|
||||
}
|
||||
[Symbol.asyncIterator](): AsyncGenerator<T, TReturn, TNext> {
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,7 @@ import assert from 'node:assert'
|
||||
import { suite, test } from 'node:test'
|
||||
import { Synchronized } from './synchronized.mjs'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (val: unknown) => {}) {
|
||||
async function* makeRangeGenerator(end = Infinity, progress = { yielded: 0 }, onYielded = (_val: unknown) => {}) {
|
||||
for (let i = 0; i < end; i++) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
yield i
|
||||
@@ -17,7 +16,7 @@ async function consume(
|
||||
iterable: AsyncGenerator,
|
||||
delay = 2500,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
onConsumed = (val: unknown, iterable: AsyncGenerator) => Promise.resolve(false)
|
||||
onConsumed = (_val: unknown, _iterable: AsyncGenerator) => Promise.resolve(false)
|
||||
) {
|
||||
for await (const val of iterable) {
|
||||
await new Promise(resolve => setTimeout(resolve, delay))
|
||||
@@ -28,6 +27,30 @@ async function consume(
|
||||
}
|
||||
|
||||
suite('success', () => {
|
||||
test('for-await-of break stops the source generator', async () => {
|
||||
let sourceClosed = false
|
||||
const source = (async function* () {
|
||||
try {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
yield i
|
||||
}
|
||||
} finally {
|
||||
sourceClosed = true
|
||||
}
|
||||
})()
|
||||
|
||||
const forker = new Synchronized(source)
|
||||
const fork = forker.fork('first')
|
||||
|
||||
// for-await-of must call fork.return() on break, which propagates to the source
|
||||
for await (const val of fork) {
|
||||
if (val === 2) break
|
||||
}
|
||||
|
||||
assert.strictEqual(sourceClosed, true)
|
||||
})
|
||||
|
||||
test('if works with multiple consumer', async () => {
|
||||
const progress = { yielded: 0 }
|
||||
const generator = makeRangeGenerator(3, progress)
|
||||
|
||||
@@ -19,7 +19,7 @@ export class Throttle {
|
||||
} else {
|
||||
speed = this.#bytesPerSecond
|
||||
}
|
||||
assert.ok(speed > 0, `speed must be greater than zero, ${speed} computed`)
|
||||
assert.ok(speed >= 0, `speed must be positive, ${speed} computed`)
|
||||
return speed
|
||||
}
|
||||
constructor(speed: number | (() => number)) {
|
||||
@@ -29,7 +29,9 @@ export class Throttle {
|
||||
getNextSlot(length: number): { timeout?: ReturnType<typeof setTimeout>; promise?: Promise<unknown> } {
|
||||
assert.notStrictEqual(length, undefined, `throttled stream need to expose a length property }`)
|
||||
assert.ok(length > 0, `throttled stream must expose a positive length property , ${length} given }`)
|
||||
|
||||
if (this.speed === 0) {
|
||||
return {}
|
||||
}
|
||||
const previous = this.#previousSlot
|
||||
const nextSlot = Math.round(previous + (length * 1000) / this.speed)
|
||||
if (nextSlot < Date.now()) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import assert from 'node:assert'
|
||||
|
||||
const TIMEOUT = Symbol('timeout')
|
||||
|
||||
export class Timeout<T> implements AsyncGenerator {
|
||||
#source: AsyncGenerator<T>
|
||||
#timeout: number
|
||||
@@ -8,21 +11,30 @@ export class Timeout<T> implements AsyncGenerator {
|
||||
this.#timeout = timeout
|
||||
}
|
||||
async next(): Promise<IteratorResult<T>> {
|
||||
let timeout: ReturnType<typeof setTimeout>
|
||||
const promiseTimeout = new Promise((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
reject(new Error('Timeout reached '))
|
||||
}, this.#timeout)
|
||||
})
|
||||
const promiseNext = new Promise((resolve, reject) => {
|
||||
this.#source.next().then(res => {
|
||||
// ensure timetout won't fire later
|
||||
clearTimeout(timeout)
|
||||
resolve(res)
|
||||
}, reject)
|
||||
})
|
||||
// promiseTimeout will never resolve
|
||||
return Promise.race([promiseNext, promiseTimeout]) as Promise<IteratorResult<T>>
|
||||
let timeoutHandle: ReturnType<typeof setTimeout>
|
||||
|
||||
const sourceNext = this.#source.next()
|
||||
const result = await Promise.race([
|
||||
sourceNext.then(res => {
|
||||
clearTimeout(timeoutHandle)
|
||||
return res
|
||||
}),
|
||||
new Promise<typeof TIMEOUT>(resolve => {
|
||||
timeoutHandle = setTimeout(() => resolve(TIMEOUT), this.#timeout)
|
||||
}),
|
||||
])
|
||||
|
||||
if (result === TIMEOUT) {
|
||||
// stop the source once the in-flight next settles to avoid data loss on the next call
|
||||
// we don't want to await for the end of timeouted code
|
||||
sourceNext.then(
|
||||
() => this.#source.return(undefined).catch(() => {}),
|
||||
() => {} // source already errored, nothing to clean up
|
||||
)
|
||||
throw new Error('Timeout reached')
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
return(): Promise<IteratorResult<T>> {
|
||||
return this.#source.return(undefined)
|
||||
|
||||
@@ -64,4 +64,24 @@ describe('Timeout class', () => {
|
||||
|
||||
await assert.rejects(timeout.throw(new Error('Test error')), 'Expected error was not thrown')
|
||||
})
|
||||
|
||||
it('should close the source generator after a timeout', async () => {
|
||||
let sourceClosed = false
|
||||
const slowSource = (async function* () {
|
||||
try {
|
||||
// resolves after 100ms so the in-flight next() eventually settles
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
yield 1
|
||||
} finally {
|
||||
sourceClosed = true
|
||||
}
|
||||
})()
|
||||
|
||||
const timeout = new Timeout(slowSource, 10) // fires before source yields
|
||||
await assert.rejects(() => timeout.next(), /Timeout reached/)
|
||||
|
||||
// wait for the in-flight next() to settle so source.return() can be called
|
||||
await new Promise(resolve => setTimeout(resolve, 200))
|
||||
assert.strictEqual(sourceClosed, true)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
"compilerOptions": {
|
||||
"target": "es2023",
|
||||
"module": "ESNext",
|
||||
"types": ["node"], // Only use types from Node.js and packages that are explicitly imported
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"declaration": true, // Generates `.d.ts` files for type safety
|
||||
|
||||
65
@vates/http-server-plus/.USAGE.md
Normal file
65
@vates/http-server-plus/.USAGE.md
Normal file
@@ -0,0 +1,65 @@
|
||||
### `create([opts], [requestListener])`
|
||||
|
||||
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
|
||||
|
||||
```js
|
||||
import { create } from '@vates/http-server-plus'
|
||||
|
||||
const server = create(function onRequest(req, res) {
|
||||
res.end('Hello')
|
||||
})
|
||||
```
|
||||
|
||||
### `server.listen(opts)`
|
||||
|
||||
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
|
||||
|
||||
```js
|
||||
// HTTP on a specific port
|
||||
await server.listen({ port: 8080 })
|
||||
|
||||
// HTTPS — detected automatically when cert+key are provided
|
||||
await server.listen({
|
||||
port: 443,
|
||||
cert: fs.readFileSync('certificate.pem'),
|
||||
key: fs.readFileSync('private_key.pem'),
|
||||
})
|
||||
|
||||
// Unix socket
|
||||
await server.listen({ socket: '/var/run/http.sock' })
|
||||
|
||||
// File descriptor
|
||||
await server.listen({ fd: 3 })
|
||||
|
||||
// Systemd socket activation
|
||||
await server.listen({ systemdSocket: 0 })
|
||||
```
|
||||
|
||||
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
|
||||
|
||||
Multiple endpoints can be started on the same instance:
|
||||
|
||||
```js
|
||||
const [httpAddr, httpsAddr] = await Promise.all([server.listen({ port: 80 }), server.listen({ port: 443, cert, key })])
|
||||
console.log('listening on', httpAddr, httpsAddr)
|
||||
```
|
||||
|
||||
### HTTP/2
|
||||
|
||||
Pass a custom `createSecureServer` factory to enable HTTP/2:
|
||||
|
||||
```js
|
||||
import { createSecureServer } from 'node:http2'
|
||||
import { create } from '@vates/http-server-plus'
|
||||
|
||||
const server = create({ createSecureServer }, app)
|
||||
await server.listen({ port: 443, cert, key })
|
||||
```
|
||||
|
||||
### `server.close()`
|
||||
|
||||
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
|
||||
|
||||
```js
|
||||
await server.close()
|
||||
```
|
||||
1
@vates/http-server-plus/.npmignore
Symbolic link
1
@vates/http-server-plus/.npmignore
Symbolic link
@@ -0,0 +1 @@
|
||||
../../scripts/npmignore
|
||||
91
@vates/http-server-plus/README.md
Normal file
91
@vates/http-server-plus/README.md
Normal file
@@ -0,0 +1,91 @@
|
||||
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
|
||||
|
||||
# @vates/http-server-plus
|
||||
|
||||
> Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance
|
||||
|
||||
## Usage
|
||||
|
||||
### `create([opts], [requestListener])`
|
||||
|
||||
Creates a new server instance. `opts` may contain `createServer` and `createSecureServer` factories (defaults to `http.createServer` and `https.createServer`). `requestListener` is registered as a `request` event listener.
|
||||
|
||||
```js
|
||||
import { create } from '@vates/http-server-plus'
|
||||
|
||||
const server = create(function onRequest(req, res) {
|
||||
res.end('Hello')
|
||||
})
|
||||
```
|
||||
|
||||
### `server.listen(opts)`
|
||||
|
||||
Starts listening. Returns a promise that resolves to a human-readable address string, or rejects with `error.niceAddress` set.
|
||||
|
||||
```js
|
||||
// HTTP on a specific port
|
||||
await server.listen({ port: 8080 })
|
||||
|
||||
// HTTPS — detected automatically when cert+key are provided
|
||||
await server.listen({
|
||||
port: 443,
|
||||
cert: fs.readFileSync('certificate.pem'),
|
||||
key: fs.readFileSync('private_key.pem'),
|
||||
})
|
||||
|
||||
// Unix socket
|
||||
await server.listen({ socket: '/var/run/http.sock' })
|
||||
|
||||
// File descriptor
|
||||
await server.listen({ fd: 3 })
|
||||
|
||||
// Systemd socket activation
|
||||
await server.listen({ systemdSocket: 0 })
|
||||
```
|
||||
|
||||
> As a convenience, if `hostname` is `'localhost'`, it will listen on both `127.0.0.1` and `::1`, similar to what Node does when no hostname is provided.
|
||||
|
||||
Multiple endpoints can be started on the same instance:
|
||||
|
||||
```js
|
||||
const [httpAddr, httpsAddr] = await Promise.all([
|
||||
server.listen({ port: 80 }),
|
||||
server.listen({ port: 443, cert, key }),
|
||||
])
|
||||
console.log('listening on', httpAddr, httpsAddr)
|
||||
```
|
||||
|
||||
### HTTP/2
|
||||
|
||||
Pass a custom `createSecureServer` factory to enable HTTP/2:
|
||||
|
||||
```js
|
||||
import { createSecureServer } from 'node:http2'
|
||||
import { create } from '@vates/http-server-plus'
|
||||
|
||||
const server = create({ createSecureServer }, app)
|
||||
await server.listen({ port: 443, cert, key })
|
||||
```
|
||||
|
||||
### `server.close()`
|
||||
|
||||
Closes all underlying servers. Returns a promise that resolves once the `close` event is emitted.
|
||||
|
||||
```js
|
||||
await server.close()
|
||||
```
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
the code.
|
||||
|
||||
You may:
|
||||
|
||||
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
|
||||
you've encountered;
|
||||
- fork and create a pull request.
|
||||
|
||||
## License
|
||||
|
||||
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)
|
||||
268
@vates/http-server-plus/index.js
Normal file
268
@vates/http-server-plus/index.js
Normal file
@@ -0,0 +1,268 @@
|
||||
'use strict'
|
||||
|
||||
// ===================================================================
|
||||
|
||||
const EventEmitter = require('events').EventEmitter
|
||||
const formatUrl = require('url').format
|
||||
const fromEvent = require('promise-toolbox/fromEvent')
|
||||
const inherits = require('util').inherits
|
||||
const resolvePath = require('path').resolve
|
||||
|
||||
// ===================================================================
|
||||
|
||||
const forwardedEvents = [
|
||||
'checkContinue',
|
||||
'checkExpectation',
|
||||
'clientError',
|
||||
'connect',
|
||||
'connection',
|
||||
'listening',
|
||||
'request',
|
||||
'secureConnection',
|
||||
'upgrade',
|
||||
]
|
||||
|
||||
function extractProperty(obj, prop) {
|
||||
const value = obj[prop]
|
||||
if (value !== undefined) {
|
||||
delete obj[prop]
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function forOwn(obj, cb) {
|
||||
const keys = Object.keys(obj)
|
||||
for (let i = 0, n = keys.length; i < n; ++i) {
|
||||
cb(obj[keys[i]])
|
||||
}
|
||||
}
|
||||
|
||||
function getSystemdFd(index) {
|
||||
if (process.pid !== +process.env.LISTEN_PID) {
|
||||
throw new Error('systemd sockets are not meant for us (LISTEN_PID)')
|
||||
}
|
||||
|
||||
const listenFds = process.env.LISTEN_FDS
|
||||
if (listenFds == null) {
|
||||
throw new Error('no systemd sockets found (LISTEN_FDS)')
|
||||
}
|
||||
const max = listenFds - 1
|
||||
|
||||
if (index > max) {
|
||||
throw new Error('no such systemd socket found')
|
||||
}
|
||||
|
||||
return index + 3 // 3 = SD_LISTEN_FDS_START
|
||||
}
|
||||
|
||||
const isEmpty = obj => Object.keys(obj).length === 0
|
||||
|
||||
// ===================================================================
|
||||
|
||||
function onNewListener(event, listener) {
|
||||
if (forwardedEvents.includes(event)) {
|
||||
forOwn(this._servers, server => {
|
||||
server.on(event, listener)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function onRemoveListener(event, listener) {
|
||||
if (forwardedEvents.includes(event)) {
|
||||
forOwn(this._servers, server => {
|
||||
server.off(event, listener)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Server(opts) {
|
||||
EventEmitter.call(this)
|
||||
|
||||
this._createServer = (opts != null && opts.createServer) || require('http').createServer
|
||||
this._createSecureServer = (opts != null && opts.createSecureServer) || require('https').createServer
|
||||
this._servers = Object.create(null)
|
||||
this._niceAddresses = Object.create(null)
|
||||
|
||||
this.on('removeListener', onRemoveListener).on('newListener', onNewListener)
|
||||
}
|
||||
inherits(Server, EventEmitter)
|
||||
|
||||
const proto = Server.prototype
|
||||
|
||||
function getAddress(serverId) {
|
||||
return this[serverId].address()
|
||||
}
|
||||
proto.addresses = function Server$addresses() {
|
||||
const servers = this._servers
|
||||
return Object.keys(servers).map(getAddress, servers)
|
||||
}
|
||||
|
||||
function getNiceAddress(serverId) {
|
||||
return this[serverId]
|
||||
}
|
||||
proto.niceAddresses = function Server$niceAddresses() {
|
||||
const niceAddresses = this._niceAddresses
|
||||
return Object.keys(niceAddresses).map(getNiceAddress, niceAddresses)
|
||||
}
|
||||
|
||||
function close(server) {
|
||||
server.close()
|
||||
}
|
||||
|
||||
proto.close = function Server$close(callback) {
|
||||
if (callback) {
|
||||
this.once('close', callback)
|
||||
}
|
||||
|
||||
// Emit the close event even if there are no registered servers.
|
||||
if (isEmpty(this._servers)) {
|
||||
const self = this
|
||||
setImmediate(function () {
|
||||
self.emit('close')
|
||||
})
|
||||
return Promise.resolve()
|
||||
}
|
||||
// Closes each servers.
|
||||
forOwn(this._servers, close)
|
||||
|
||||
return fromEvent(this, 'close')
|
||||
}
|
||||
|
||||
let nextId = 0
|
||||
proto.listen = function Server$listen(opts) {
|
||||
opts = Object.assign({}, opts)
|
||||
|
||||
if (opts.hostname === 'localhost') {
|
||||
return Promise.all([
|
||||
this.listen(Object.assign(opts, { hostname: '::1' })),
|
||||
this.listen(Object.assign(opts, { hostname: '127.0.0.1' })),
|
||||
]).then(addresses => addresses[0])
|
||||
}
|
||||
|
||||
const hostname = extractProperty(opts, 'hostname')
|
||||
let port = extractProperty(opts, 'port')
|
||||
const systemdSocket = extractProperty(opts, 'systemdSocket')
|
||||
let fd = extractProperty(opts, 'fd')
|
||||
let socket = extractProperty(opts, 'socket')
|
||||
|
||||
const servers = this._servers
|
||||
|
||||
const niceAddresses = this._niceAddresses
|
||||
|
||||
let server, protocol
|
||||
if (opts.pfx || opts.SNICallback || (opts.cert && opts.key)) {
|
||||
server = this._createSecureServer(opts)
|
||||
protocol = 'https'
|
||||
} else {
|
||||
server = this._createServer(opts)
|
||||
protocol = 'http'
|
||||
}
|
||||
|
||||
const id = nextId++
|
||||
servers[id] = server
|
||||
|
||||
if (systemdSocket != null) {
|
||||
fd = getSystemdFd(systemdSocket)
|
||||
}
|
||||
|
||||
// Compute a temporary nice address to display in case of error.
|
||||
let niceAddress
|
||||
if (fd != null) {
|
||||
server.listen({ fd })
|
||||
niceAddress = protocol + '://<fd:' + fd + '>'
|
||||
} else if (socket != null) {
|
||||
socket = resolvePath(socket)
|
||||
server.listen(socket)
|
||||
niceAddress = protocol + '://' + socket
|
||||
} else {
|
||||
if (port == null) {
|
||||
port = protocol === 'https' ? 443 : 80
|
||||
}
|
||||
server.listen(port, hostname)
|
||||
niceAddress = formatUrl({
|
||||
protocol,
|
||||
|
||||
// Hostname default to localhost.
|
||||
hostname: hostname || 'localhost',
|
||||
|
||||
// No port means random, unknown for now.
|
||||
port: port || '<unknown>',
|
||||
})
|
||||
}
|
||||
|
||||
const emit = this.emit.bind(this)
|
||||
server.once('close', function onClose() {
|
||||
delete servers[id]
|
||||
delete niceAddresses[id]
|
||||
|
||||
if (isEmpty(servers)) {
|
||||
emit('close')
|
||||
}
|
||||
})
|
||||
|
||||
server.on('error', function onError() {
|
||||
delete servers[id]
|
||||
|
||||
// FIXME: Should it be forwarded and be fatal if there are no
|
||||
// listeners?
|
||||
})
|
||||
|
||||
forwardedEvents.forEach(event => {
|
||||
this.listeners(event).forEach(listener => {
|
||||
server.on(event, listener)
|
||||
})
|
||||
})
|
||||
|
||||
return fromEvent(server, 'listening').then(
|
||||
function () {
|
||||
const address = server.address()
|
||||
const niceAddress =
|
||||
typeof address === 'string'
|
||||
? protocol + '://' + address
|
||||
: formatUrl({
|
||||
protocol,
|
||||
hostname: address.address,
|
||||
port: address.port,
|
||||
})
|
||||
niceAddresses[id] = niceAddress
|
||||
return niceAddress
|
||||
},
|
||||
function (error) {
|
||||
error.niceAddress = niceAddress
|
||||
throw error
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function ref(server) {
|
||||
server.ref()
|
||||
}
|
||||
|
||||
proto.ref = function Server$ref() {
|
||||
forOwn(this._servers, ref)
|
||||
}
|
||||
|
||||
function unref(server) {
|
||||
server.unref()
|
||||
}
|
||||
|
||||
proto.unref = function Server$unref() {
|
||||
forOwn(this._servers, unref)
|
||||
}
|
||||
|
||||
// ===================================================================
|
||||
|
||||
module.exports = exports = Server
|
||||
|
||||
exports.create = function create(opts, requestListener) {
|
||||
if (typeof opts === 'function') {
|
||||
requestListener = opts
|
||||
opts = undefined
|
||||
}
|
||||
|
||||
const server = new Server(opts)
|
||||
if (requestListener !== undefined) {
|
||||
server.on('request', requestListener)
|
||||
}
|
||||
return server
|
||||
}
|
||||
303
@vates/http-server-plus/index.spec.js
Normal file
303
@vates/http-server-plus/index.spec.js
Normal file
@@ -0,0 +1,303 @@
|
||||
'use strict'
|
||||
|
||||
// ===================================================================
|
||||
|
||||
const assert = require('assert/strict')
|
||||
const { EventEmitter } = require('events')
|
||||
const resolvePath = require('path').resolve
|
||||
const { describe, it } = require('node:test')
|
||||
|
||||
const HttpServerPlus = require('./')
|
||||
|
||||
// ===================================================================
|
||||
|
||||
// Creates a minimal mock server that can be used as a drop-in for http/https.Server.
|
||||
// `address` is the value returned by mock.address() — use an object for TCP, a string for sockets.
|
||||
function createMock(address) {
|
||||
const mock = new EventEmitter()
|
||||
mock.listen = function () {
|
||||
setImmediate(() => this.emit('listening'))
|
||||
}
|
||||
mock.address = () => address
|
||||
mock.close = function () {
|
||||
setImmediate(() => this.emit('close'))
|
||||
}
|
||||
return mock
|
||||
}
|
||||
|
||||
// ===================================================================
|
||||
|
||||
describe('HttpServerPlus', function () {
|
||||
describe('#create()', function () {
|
||||
it('creates a new instance', function () {
|
||||
assert(HttpServerPlus.create() instanceof HttpServerPlus)
|
||||
})
|
||||
|
||||
it('can register a `request` listener', function () {
|
||||
const listener = function () {}
|
||||
const server = HttpServerPlus.create(listener)
|
||||
|
||||
assert.deepEqual(server.listeners('request'), [listener])
|
||||
})
|
||||
})
|
||||
|
||||
describe('.addresses()', function () {
|
||||
it('returns the list of addresses the server is listening at', function () {
|
||||
const addr = { address: '127.0.0.1', port: 8080, family: 'IPv4' }
|
||||
const mock = createMock(addr)
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ port: 8080 }).then(function () {
|
||||
assert.deepEqual(server.addresses(), [addr])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('.niceAddresses()', function () {
|
||||
it('returns the list of human readable addresses the server is listening at', function () {
|
||||
const mock = createMock({ address: '127.0.0.1', port: 8080 })
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function (address) {
|
||||
assert.deepEqual(server.niceAddresses(), [address])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('secureConnection event forwarding', function () {
|
||||
function createMockHttpsServer() {
|
||||
const server = new EventEmitter()
|
||||
server.listen = function () {
|
||||
setImmediate(() => this.emit('listening'))
|
||||
}
|
||||
server.address = () => ({ address: '127.0.0.1', port: 8443 })
|
||||
server.close = function () {
|
||||
setImmediate(() => this.emit('close'))
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
function createServerWithMock() {
|
||||
let mockServer
|
||||
const server = HttpServerPlus.create({
|
||||
createSecureServer: () => {
|
||||
mockServer = createMockHttpsServer()
|
||||
return mockServer
|
||||
},
|
||||
})
|
||||
return {
|
||||
server,
|
||||
getMock: () => mockServer,
|
||||
}
|
||||
}
|
||||
|
||||
it('forwards secureConnection to listeners registered before listen()', function () {
|
||||
const { server, getMock } = createServerWithMock()
|
||||
|
||||
const received = []
|
||||
server.on('secureConnection', socket => received.push(socket))
|
||||
|
||||
const fakeSocket = {}
|
||||
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
|
||||
getMock().emit('secureConnection', fakeSocket)
|
||||
assert.deepEqual(received, [fakeSocket])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('forwards secureConnection to listeners registered after listen()', function () {
|
||||
const { server, getMock } = createServerWithMock()
|
||||
|
||||
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
|
||||
const received = []
|
||||
server.on('secureConnection', socket => received.push(socket))
|
||||
|
||||
const fakeSocket = {}
|
||||
getMock().emit('secureConnection', fakeSocket)
|
||||
assert.deepEqual(received, [fakeSocket])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('stops forwarding secureConnection after listener is removed', function () {
|
||||
const { server, getMock } = createServerWithMock()
|
||||
|
||||
const received = []
|
||||
const listener = socket => received.push(socket)
|
||||
server.on('secureConnection', listener)
|
||||
|
||||
return server.listen({ cert: 'fake-cert', key: 'fake-key' }).then(function () {
|
||||
server.off('secureConnection', listener)
|
||||
|
||||
getMock().emit('secureConnection', {})
|
||||
assert.deepEqual(received, [])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('.listen()', function () {
|
||||
it('can use a host:port', function () {
|
||||
let listenArgs
|
||||
const mock = createMock({ address: '127.0.0.1', port: 8080 })
|
||||
mock.listen = function (...args) {
|
||||
listenArgs = args
|
||||
setImmediate(() => this.emit('listening'))
|
||||
}
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ hostname: '127.0.0.1', port: 8080 }).then(function () {
|
||||
assert.deepEqual(listenArgs, [8080, '127.0.0.1'])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('can use a systemd socket', function () {
|
||||
const SD_LISTEN_FDS_START = 3
|
||||
const SOCKET_INDEX = 0
|
||||
|
||||
let listenArgs
|
||||
const mock = createMock('http://<fd:3>')
|
||||
mock.listen = function (...args) {
|
||||
listenArgs = args
|
||||
setImmediate(() => this.emit('listening'))
|
||||
}
|
||||
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
const env = process.env
|
||||
env.LISTEN_FDS = String(SOCKET_INDEX + 1)
|
||||
env.LISTEN_PID = String(process.pid)
|
||||
|
||||
return server.listen({ systemdSocket: SOCKET_INDEX }).then(function () {
|
||||
delete env.LISTEN_FDS
|
||||
delete env.LISTEN_PID
|
||||
|
||||
assert.deepEqual(listenArgs, [{ fd: SOCKET_INDEX + SD_LISTEN_FDS_START }])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('can use a socket', function () {
|
||||
const socketPath = '/tmp/http-server-plus-test.sock'
|
||||
let listenArgs
|
||||
const mock = createMock(socketPath)
|
||||
mock.listen = function (...args) {
|
||||
listenArgs = args
|
||||
setImmediate(() => this.emit('listening'))
|
||||
}
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ socket: socketPath }).then(function () {
|
||||
assert.deepEqual(listenArgs, [resolvePath(socketPath)])
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('can use a HTTPS certificate', function () {
|
||||
let capturedOpts
|
||||
const mock = createMock({ address: '127.0.0.1', port: 8443 })
|
||||
const server = HttpServerPlus.create({
|
||||
createSecureServer: opts => {
|
||||
capturedOpts = opts
|
||||
return mock
|
||||
},
|
||||
})
|
||||
|
||||
return server.listen({ cert: 'my-cert', key: 'my-key', port: 8443 }).then(function (address) {
|
||||
assert.equal(capturedOpts.cert, 'my-cert')
|
||||
assert.equal(capturedOpts.key, 'my-key')
|
||||
assert.ok(address.startsWith('https://'), `expected https URL, got: ${address}`)
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('returns a promise which will resolve once listening', function () {
|
||||
const mock = createMock({ address: '127.0.0.1', port: 8080 })
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ port: 8080 }).then(function (address) {
|
||||
assert.equal(typeof address, 'string')
|
||||
assert.ok(address.startsWith('http://'), `expected http URL, got: ${address}`)
|
||||
return server.close()
|
||||
})
|
||||
})
|
||||
|
||||
it('returns a promise which will reject if failure', function () {
|
||||
const listenError = new Error('EADDRINUSE')
|
||||
const mock = createMock(null)
|
||||
mock.listen = function () {
|
||||
setImmediate(() => this.emit('error', listenError))
|
||||
}
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
return server.listen({ port: 8080 }).then(
|
||||
function () {
|
||||
assert.fail('should have rejected')
|
||||
},
|
||||
function (err) {
|
||||
assert.equal(err, listenError)
|
||||
assert.equal(typeof err.niceAddress, 'string')
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('.close()', function () {
|
||||
it('closes all servers', function () {
|
||||
const closed = []
|
||||
let callCount = 0
|
||||
|
||||
const server = HttpServerPlus.create({
|
||||
createServer: () => {
|
||||
const id = callCount++
|
||||
const mock = createMock({ address: '127.0.0.1', port: id })
|
||||
mock.close = function () {
|
||||
closed.push(id)
|
||||
setImmediate(() => this.emit('close'))
|
||||
}
|
||||
return mock
|
||||
},
|
||||
})
|
||||
|
||||
return Promise.all([server.listen({ port: 0 }), server.listen({ port: 0 })])
|
||||
.then(function () {
|
||||
return server.close()
|
||||
})
|
||||
.then(function () {
|
||||
assert.equal(closed.length, 2)
|
||||
})
|
||||
})
|
||||
|
||||
it('emit the `close` event when all servers are closed', function () {
|
||||
const mock = createMock({ address: '127.0.0.1', port: 8080 })
|
||||
const server = HttpServerPlus.create({ createServer: () => mock })
|
||||
|
||||
let closeEmitted = false
|
||||
server.on('close', () => {
|
||||
closeEmitted = true
|
||||
})
|
||||
|
||||
return server
|
||||
.listen({ port: 8080 })
|
||||
.then(function () {
|
||||
return server.close()
|
||||
})
|
||||
.then(function () {
|
||||
assert.ok(closeEmitted)
|
||||
})
|
||||
})
|
||||
|
||||
it('emit the `close` event even if there are no servers', function () {
|
||||
const server = HttpServerPlus.create()
|
||||
|
||||
// close() with no servers returns Promise.resolve() immediately (before emitting
|
||||
// 'close' via setImmediate), so we must wait for the event itself, not the promise.
|
||||
return new Promise(function (resolve) {
|
||||
server.close(resolve)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
37
@vates/http-server-plus/package.json
Normal file
37
@vates/http-server-plus/package.json
Normal file
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "@vates/http-server-plus",
|
||||
"version": "2.0.0",
|
||||
"license": "ISC",
|
||||
"description": "Augmented `http.Server`, HTTP/HTTPS/HTTP2 and multiple ports on the same instance",
|
||||
"keywords": [
|
||||
"http",
|
||||
"https",
|
||||
"server",
|
||||
"ports"
|
||||
],
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/http-server-plus",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@vates/http-server-plus",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"dependencies": {
|
||||
"promise-toolbox": "^0.21.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint-plugin-n": "^15.1.0"
|
||||
},
|
||||
"scripts": {
|
||||
"pretest": "eslint .",
|
||||
"test": "node --test index.spec.js"
|
||||
},
|
||||
"private": true,
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,17 @@ export class NbdDisk extends RandomAccessDisk {
|
||||
if (!this.#nbdClient) {
|
||||
throw new Error("can't readBlock before init")
|
||||
}
|
||||
const data = await this.#nbdClient.readBlock(index, this.getBlockSize())
|
||||
let data = await this.#nbdClient.readBlock(index, this.getBlockSize())
|
||||
if (data.length !== this.getBlockSize()) {
|
||||
// this condition only work on non aligned disks
|
||||
// we won't accept truncated block from unaligned disks
|
||||
if (index === Math.ceil(this.getVirtualSize() / this.getBlockSize()) - 1) {
|
||||
// last block add zeros at the end
|
||||
data = Buffer.concat([data], this.getBlockSize())
|
||||
} else {
|
||||
throw new Error(`Block ${index} is not at the right size, expecting ${this.getBlockSize()}, got ${data.length}`)
|
||||
}
|
||||
}
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
@@ -117,6 +127,6 @@ export class NbdDisk extends RandomAccessDisk {
|
||||
}
|
||||
const blockStart = index * this.getBlockSize()
|
||||
const blockEnd = (index + 1) * this.getBlockSize()
|
||||
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && blockEnd >= offset)
|
||||
return this.#dataMap.some(({ offset, length }) => offset + length > blockStart && offset < blockEnd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,6 +353,7 @@ export default class NbdClient {
|
||||
async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
|
||||
return pRetry(() => this.#readBlock(index, size), {
|
||||
tries: this.#readBlockRetries,
|
||||
when: error => error.code !== 'ERR_ABORTED',
|
||||
onRetry: async err => {
|
||||
warn('will retry reading block ', index, err)
|
||||
await this.reconnect()
|
||||
@@ -368,9 +369,9 @@ export default class NbdClient {
|
||||
* A promise that resolves to an array where each object represents a segment:
|
||||
* - `offset` — The byte offset from the start.
|
||||
* - `length` — The size of the segment in bytes.
|
||||
* - `type` — A numeric code indicating the segment type ( 0 means no data).
|
||||
* - `type` — A numeric code indicating the segment type ( 0 means data).
|
||||
*/
|
||||
/* async */ getMap() {
|
||||
/* async */ getMap(signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const process = spawn('nbdinfo', [
|
||||
'--json',
|
||||
@@ -381,8 +382,17 @@ export default class NbdClient {
|
||||
let errText = ''
|
||||
process.stdout.on('data', data => (text += data))
|
||||
process.stderr.on('data', data => (errText += data))
|
||||
process.on('error', reject)
|
||||
const onAbort = () => {
|
||||
process.kill()
|
||||
reject(signal.reason)
|
||||
}
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
process.on('error', err => {
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
reject(err)
|
||||
})
|
||||
process.on('close', code => {
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
if (code !== 0) {
|
||||
return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
|
||||
}
|
||||
|
||||
@@ -138,8 +138,8 @@ export default class MultiNbdClient {
|
||||
* - `length` — The size of the segment in bytes.
|
||||
* - `type` — A numeric code indicating the segment type (0 means no data).
|
||||
*/
|
||||
async getMap() {
|
||||
async getMap(signal) {
|
||||
// ask the map from one of the connected client
|
||||
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap()
|
||||
return this.#clients[Math.floor(this.#clients.length * Math.random())].getMap(signal)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,19 +13,19 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "3.2.1",
|
||||
"version": "3.4.0",
|
||||
"engines": {
|
||||
"node": ">=14.0"
|
||||
},
|
||||
"main": "./index.mjs",
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@vates/read-chunk": "^1.2.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/disk-transform": "^1.0.1",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/disk-transform": "^1.2.3",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"xen-api": "^4.7.4"
|
||||
"xen-api": "^4.7.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tap": "^18.7.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@vates/node-vsphere-soap",
|
||||
"version": "2.1.2",
|
||||
"version": "2.1.3",
|
||||
"description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
|
||||
"main": "lib/client.mjs",
|
||||
"author": "reedog117",
|
||||
@@ -11,8 +11,8 @@
|
||||
},
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"axios": "^1.8.0",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"axios": "^1.15.0",
|
||||
"soap": "^1.0.0",
|
||||
"soap-cookie": "^0.10.1"
|
||||
},
|
||||
|
||||
@@ -23,7 +23,7 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
|
||||
Object.defineProperty(taskLog, '$root', { value: taskLog })
|
||||
|
||||
// start of a root task
|
||||
onRootTaskStart(taskLog)
|
||||
onRootTaskStart(taskLog, event)
|
||||
} else {
|
||||
// start of a subtask
|
||||
const parent = taskLogs.get(parentId)
|
||||
@@ -54,10 +54,10 @@ exports.makeOnProgress = function ({ onRootTaskEnd = noop, onRootTaskStart = noo
|
||||
}
|
||||
|
||||
if (type === 'end' && taskLog.$root === taskLog) {
|
||||
onRootTaskEnd(taskLog)
|
||||
onRootTaskEnd(taskLog, event)
|
||||
}
|
||||
}
|
||||
|
||||
onTaskUpdate(taskLog)
|
||||
onTaskUpdate(taskLog, event)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,27 @@ function alreadyEnded() {
|
||||
throw new Error('task has already ended')
|
||||
}
|
||||
|
||||
const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
|
||||
|
||||
// Create a serializable object from an error.
|
||||
//
|
||||
// Otherwise some fields might be non-enumerable and missing from logs.
|
||||
const serializeError = error => {
|
||||
return error instanceof Error
|
||||
? {
|
||||
...error, // Copy enumerable properties.
|
||||
cause: error.cause,
|
||||
code: error.code,
|
||||
errors: serializeErrors(error.errors), // supports AggregateError
|
||||
message: error.message,
|
||||
name: error.name,
|
||||
stack: error.stack,
|
||||
}
|
||||
: error
|
||||
}
|
||||
|
||||
exports.serializeError = serializeError
|
||||
|
||||
// define a read-only, non-enumerable, non-configurable property
|
||||
function define(object, property, value) {
|
||||
Object.defineProperty(object, property, { value })
|
||||
@@ -72,7 +93,11 @@ class Task {
|
||||
_onProgress
|
||||
|
||||
get id() {
|
||||
return (this.id = Math.random().toString(36).slice(2))
|
||||
// Use a compact, sortable, string representation of the creation date,
|
||||
// and add a random part to differentiate tasks created at the same time
|
||||
//
|
||||
// Due to the padding, dates are sortable up to 5188-04-22T11:04:28.415Z
|
||||
return (this.id = Date.now().toString(36).padStart(9, '0') + '-' + Math.random().toString(36).slice(2))
|
||||
}
|
||||
set id(value) {
|
||||
define(this, 'id', value)
|
||||
@@ -136,6 +161,10 @@ class Task {
|
||||
assert.equal(this.#status, PENDING)
|
||||
assert.equal(this.#running, false)
|
||||
|
||||
if (result instanceof Error && result.toJSON === undefined) {
|
||||
result = serializeError(result)
|
||||
}
|
||||
|
||||
this.#emit('end', { status, result })
|
||||
this._onProgress = alreadyEnded
|
||||
this.#status = status
|
||||
|
||||
@@ -4,7 +4,7 @@ const assert = require('node:assert').strict
|
||||
const { describe, it } = require('node:test')
|
||||
|
||||
const { makeOnProgress } = require('./combineEvents.js')
|
||||
const { Task } = require('./index.js')
|
||||
const { Task, serializeError } = require('./index.js')
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -28,6 +28,36 @@ function createTask(opts) {
|
||||
return task
|
||||
}
|
||||
|
||||
describe('serializeError', function () {
|
||||
it('serializes simple error', function () {
|
||||
const err = new Error('test Error')
|
||||
err.code = 404
|
||||
const serialized = serializeError(err)
|
||||
assert.equal(serialized.message, 'test Error')
|
||||
assert.equal(serialized.name, 'Error')
|
||||
assert.equal(serialized.code, 404)
|
||||
assert.notEqual(serialized.stack, undefined)
|
||||
})
|
||||
|
||||
it('serializes aggregate error', function () {
|
||||
const err = new AggregateError([new Error('test error 1'), new Error('test error 2')], 'Error message', {
|
||||
cause: 'broken',
|
||||
})
|
||||
const serialized = serializeError(err)
|
||||
assert.equal(serialized.message, 'Error message')
|
||||
assert.equal(serialized.name, 'AggregateError')
|
||||
assert.equal(serialized.cause, 'broken')
|
||||
assert.notEqual(serialized.stack, undefined)
|
||||
assert.equal(serialized.errors.length, 2)
|
||||
assert.equal(serialized.errors[0].name, 'Error')
|
||||
assert.equal(serialized.errors[1].name, 'Error')
|
||||
assert.equal(serialized.errors[0].message, 'test error 1')
|
||||
assert.equal(serialized.errors[1].message, 'test error 2')
|
||||
assert.notEqual(serialized.errors[0].stack, undefined)
|
||||
assert.notEqual(serialized.errors[1].stack, undefined)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Task', function () {
|
||||
describe('constructor', function () {
|
||||
it('data properties are passed to the start event', async function () {
|
||||
@@ -106,7 +136,7 @@ describe('Task', function () {
|
||||
assert.equal(task.$events.length, 3)
|
||||
assertEvent(task, { type: 'start' }, 0)
|
||||
assertEvent(task, { type: 'abortionRequested', reason }, 1)
|
||||
assertEvent(task, { type: 'end', status: 'failure', result }, 2)
|
||||
assertEvent(task, { type: 'end', status: 'failure', result: serializeError(result) }, 2)
|
||||
})
|
||||
|
||||
it('does not abort if the task succeed', async function () {
|
||||
@@ -176,7 +206,7 @@ describe('Task', function () {
|
||||
|
||||
assertEvent(task, {
|
||||
status: 'failure',
|
||||
result: error,
|
||||
result: serializeError(error),
|
||||
type: 'end',
|
||||
})
|
||||
})
|
||||
@@ -475,7 +505,7 @@ describe('Task', function () {
|
||||
assert.equal(task.status, 'failure')
|
||||
assertEvent(task, {
|
||||
status: 'failure',
|
||||
result: e,
|
||||
result: serializeError(e),
|
||||
type: 'end',
|
||||
})
|
||||
})
|
||||
@@ -511,7 +541,7 @@ describe('Task', function () {
|
||||
assert.equal(task.status, 'failure')
|
||||
assertEvent(task, {
|
||||
status: 'failure',
|
||||
result: e,
|
||||
result: serializeError(e),
|
||||
type: 'end',
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "0.6.2",
|
||||
"version": "0.7.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@vates/types",
|
||||
"private": false,
|
||||
"version": "1.12.0",
|
||||
"version": "1.23.0",
|
||||
"main": "./dist/index.mjs",
|
||||
"exports": {
|
||||
".": {
|
||||
@@ -9,6 +9,7 @@
|
||||
"types": "./dist/index.d.mts"
|
||||
},
|
||||
"./xo": "./dist/xo.d.mts",
|
||||
"./xo-app": "./dist/xo-app.d.mts",
|
||||
"./xen-api": "./dist/xen-api.d.mts",
|
||||
"./common": {
|
||||
"default": "./dist/common.mjs",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export * from './common.mjs'
|
||||
export * from './xen-api.mjs'
|
||||
export * from './xo.mjs'
|
||||
export * from './xo-app.mjs'
|
||||
export * from './lib/xen-orchestra-xapi.mjs'
|
||||
export * from './lib/complex-matcher.mjs'
|
||||
|
||||
@@ -1,28 +1,42 @@
|
||||
import {
|
||||
WrappedXenApiRecord,
|
||||
XenApiHost,
|
||||
XenApiHostWrapped,
|
||||
XenApiNetwork,
|
||||
XenApiNetworkWrapped,
|
||||
XenApiRecord,
|
||||
XenApiSr,
|
||||
XenApiTask,
|
||||
XenApiVbd,
|
||||
XenApiVdi,
|
||||
XenApiVif,
|
||||
XenApiVm,
|
||||
XenApiVmWrapped,
|
||||
XenApiVtpm,
|
||||
} from '../xen-api.mjs'
|
||||
import type {
|
||||
BOND_MODE,
|
||||
OPAQUE_REF_NULL,
|
||||
SUPPORTED_VDI_FORMAT,
|
||||
VBD_MODE,
|
||||
VBD_TYPE,
|
||||
VDI_TYPE,
|
||||
VIF_LOCKING_MODE,
|
||||
} from '../common.mjs'
|
||||
import type { PassThrough, Readable } from 'node:stream'
|
||||
import type {
|
||||
XoGpuGroup,
|
||||
XoVgpuType,
|
||||
XoHost,
|
||||
XoNetwork,
|
||||
XoPif,
|
||||
XoSr,
|
||||
XoUser,
|
||||
XoVdi,
|
||||
XoVgpuType,
|
||||
XoVm,
|
||||
XoVmTemplate,
|
||||
XoVif,
|
||||
} from '../xo.mjs'
|
||||
import type { SUPPORTED_VDI_FORMAT } from '../common.mjs'
|
||||
|
||||
export type XcpPatches = {
|
||||
changelog?: {
|
||||
@@ -57,11 +71,13 @@ export interface Xapi {
|
||||
call: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
callAsync: <ReturnType>(...args: unknown[]) => Promise<ReturnType>
|
||||
|
||||
barrier<T extends XenApiRecord>(ref: T['$ref']): Promise<Extract<WrappedXenApiRecord, { $ref: T['$ref'] }>>
|
||||
getField<T extends XenApiRecord, K extends keyof T>(
|
||||
type: Extract<WrappedXenApiRecord, T>['$type'],
|
||||
ref: T['$ref'],
|
||||
field: K
|
||||
): Promise<T[K]>
|
||||
connectVif(vifId: XoVif['id']): Promise<void>
|
||||
createNetwork(
|
||||
params:
|
||||
| {
|
||||
@@ -78,12 +94,58 @@ export interface Xapi {
|
||||
vlan: number
|
||||
}
|
||||
): Promise<XenApiNetworkWrapped>
|
||||
createBondedNetwork(params: {
|
||||
bondMode: BOND_MODE
|
||||
pifIds: XoPif['id'][]
|
||||
name: string
|
||||
description?: string
|
||||
mtu?: number
|
||||
}): Promise<XenApiNetworkWrapped>
|
||||
deleteNetwork(id: XoNetwork['id']): Promise<void>
|
||||
deleteVif(vifId: XoVif['id']): Promise<void>
|
||||
disconnectVif(vifId: XoVif['id']): Promise<void>
|
||||
exportVmOva(vmRef: XenApiVm['$ref']): Promise<PassThrough>
|
||||
migrateVm(
|
||||
vmId: XoVm['id'],
|
||||
hostXapi: Xapi,
|
||||
hostId: XoHost['id'],
|
||||
opts?: {
|
||||
force?: boolean
|
||||
mapVdisSrs?: Record<XoVdi['id'], XoSr['id']>
|
||||
mapVifsNetworks?: Record<XoVif['id'], XoNetwork['id']>
|
||||
migrationNetworkId?: XoNetwork['id']
|
||||
sr?: XoSr['id']
|
||||
bypassAssert?: boolean
|
||||
}
|
||||
): Promise<void>
|
||||
listMissingPatches(host: XoHost['id']): Promise<XcpPatches[] | XsPatches[]>
|
||||
pool_emergencyShutdown(): Promise<void>
|
||||
resumeVm(id: XoVm['id']): Promise<void>
|
||||
unpauseVm(id: XoVm['id']): Promise<void>
|
||||
cloneVm(
|
||||
vmId: XoVm['id'],
|
||||
opts?: {
|
||||
nameLabel?: string
|
||||
fast?: boolean
|
||||
}
|
||||
): Promise<XenApiVmWrapped>
|
||||
copyVm(
|
||||
vmId: XoVm['id'],
|
||||
opts?: {
|
||||
nameLabel?: string
|
||||
srOrSrId?: XoSr['id'] | XoSr
|
||||
}
|
||||
): Promise<XenApiVmWrapped>
|
||||
remoteCopyVm(
|
||||
vmId: XoVm['id'],
|
||||
targetXapi: Xapi,
|
||||
targetSrId: XoSr['id'],
|
||||
opts?: {
|
||||
compress?: 'zstd' | 'gzip' | boolean
|
||||
nameLabel?: string
|
||||
}
|
||||
): Promise<{ vm: XenApiVmWrapped }>
|
||||
forgetSr(id: XoSr['id']): Promise<void>
|
||||
SR_importVdi(
|
||||
ref: XenApiSr['$ref'],
|
||||
stream: Readable,
|
||||
@@ -101,6 +163,7 @@ export interface Xapi {
|
||||
xenstore_data?: Record<string, string>
|
||||
}
|
||||
): Promise<XenApiVdi['$ref']>
|
||||
SR_reclaimSpace(ref: XenApiSr['$ref']): Promise<void>
|
||||
startVm(
|
||||
id: XoVm['id'],
|
||||
opts?: {
|
||||
@@ -130,6 +193,7 @@ export interface Xapi {
|
||||
metadataVm: {
|
||||
affinityHost?: XoHost['id']
|
||||
autoPoweron?: boolean
|
||||
cpus?: number
|
||||
memory?: number
|
||||
name_description?: string
|
||||
name_label: string
|
||||
@@ -171,17 +235,79 @@ export interface Xapi {
|
||||
vgpuType?: XoVgpuType['id']
|
||||
gpuGroup?: XoGpuGroup['id']
|
||||
copyHostBiosStrings?: boolean
|
||||
hvmBootFirmware?: 'uefi' | 'bios'
|
||||
secureBoot?: boolean
|
||||
},
|
||||
checkLimits?: boolean,
|
||||
creatorId?: XoUser['id'],
|
||||
opts?: { destroyAllVifs: boolean }
|
||||
): Promise<XenApiVmWrapped>
|
||||
moveVdi(
|
||||
vdiId: XenApiVdi['$ref'] | XoVdi['id'],
|
||||
srId: XenApiSr['$ref'] | XoSr['id'],
|
||||
opts?: { _failOnCbtError?: boolean }
|
||||
): Promise<XenApiVdi>
|
||||
VBD_create(params: {
|
||||
bootable?: boolean
|
||||
empty?: boolean
|
||||
mode?: VBD_MODE
|
||||
other_config?: Record<string, string>
|
||||
qos_algorithm_params?: Record<string, string>
|
||||
qos_algorithm_type?: string
|
||||
type?: VBD_TYPE
|
||||
unpluggable?: boolean
|
||||
userdevice?: string
|
||||
VDI: XenApiVdi['$ref'] | OPAQUE_REF_NULL
|
||||
VM: XenApiVm['$ref']
|
||||
}): Promise<XenApiVbd['$ref']>
|
||||
VBD_destroy(vbdRef: XenApiVbd['$ref']): Promise<void>
|
||||
VBD_unplug(vbdRef: XenApiVbd['$ref']): Promise<void>
|
||||
VDI_create(
|
||||
options: {
|
||||
name_description?: XoVdi['name_description']
|
||||
name_label?: XoVdi['name_label']
|
||||
other_config?: XoVdi['other_config']
|
||||
read_only?: boolean
|
||||
sharable?: boolean
|
||||
SR?: XenApiSr['$ref']
|
||||
tags?: XoVdi['tags']
|
||||
type?: VDI_TYPE
|
||||
virtual_size: XoVdi['size']
|
||||
xenstore_data?: Record<string, string>
|
||||
},
|
||||
extraOptions?: {
|
||||
sm_config?: Record<string, string>
|
||||
}
|
||||
): Promise<XenApiVdi['$ref']>
|
||||
VDI_destroy(vdiRef: XenApiVdi['$ref']): Promise<void>
|
||||
VDI_destroyCloudInitConfig(vdiRef: XenApiVdi['$ref'], opts?: { timeLimit?: number }): Promise<void>
|
||||
VDI_exportContent(
|
||||
vdiRef: XenApiVdi['$ref'],
|
||||
opts: { baseRef?: string; cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
|
||||
): Promise<Readable & { length?: number }>
|
||||
VDI_importContent(
|
||||
vdiRef: XenApiVdi['$ref'],
|
||||
stream: Readable,
|
||||
opts: { cancelToken?: unknown; format: SUPPORTED_VDI_FORMAT }
|
||||
): Promise<void>
|
||||
VIF_create(
|
||||
options: {
|
||||
currently_attached?: boolean
|
||||
device?: string
|
||||
ipv4_allowed?: string[]
|
||||
ipv6_allowed?: string[]
|
||||
locking_mode?: VIF_LOCKING_MODE
|
||||
MTU?: number
|
||||
network: XenApiNetwork['$ref']
|
||||
other_config?: Record<string, string>
|
||||
qos_algorithm_params?: Record<string, string>
|
||||
qos_algorithm_type?: string
|
||||
VM: XenApiVm['$ref']
|
||||
},
|
||||
extraOptions?: {
|
||||
MAC?: string
|
||||
}
|
||||
): Promise<XenApiVif['$ref']>
|
||||
VM_createCloudInitConfig(
|
||||
vmRef: XenApiVm['$ref'],
|
||||
cloudConfig: string,
|
||||
@@ -200,5 +326,16 @@ export interface Xapi {
|
||||
pathname: string,
|
||||
params?: { host?: XenApiHost; query?: Record<string, unknown>; task?: boolean | XenApiTask['$ref'] }
|
||||
): Promise<{ body: Readable }>
|
||||
clearHost(host: Pick<XenApiHostWrapped, '$ref' | '$pool'>, force?: boolean): Promise<void>
|
||||
disableHost(hostId: XoHost['id']): Promise<void>
|
||||
enableHost(hostId: XoHost['id']): Promise<void>
|
||||
getRecordByUuid<
|
||||
Type extends WrappedXenApiRecord['$type'],
|
||||
XenApiRecord extends WrappedXenApiRecord = Extract<WrappedXenApiRecord, { $type: Type }>,
|
||||
>(
|
||||
type: Type,
|
||||
uuid: XenApiRecord['uuid']
|
||||
): Promise<XenApiRecord>
|
||||
isHyperThreadingEnabled(hostId: XoHost['id']): Promise<boolean | null>
|
||||
VTPM_create(params: { VM: XenApiVm['$ref']; is_unique?: boolean; contents?: string }): Promise<XenApiVtpm['$ref']>
|
||||
}
|
||||
|
||||
@@ -70,6 +70,11 @@ import type { Xapi } from './lib/xen-orchestra-xapi.mjs'
|
||||
// https://github.com/vatesfr/xen-orchestra/tree/wip-xapi-types-generator/%40xen-orchestra/xapi-generator
|
||||
// All properties added after XenServer 7.0 (dundee) release have been marked as optional
|
||||
|
||||
type TagCallMethods = {
|
||||
(method: 'add_tags', tag: string): Promise<void>
|
||||
(method: 'remove_tags', tag: string): Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Add properties injected by `xen-api`.
|
||||
* $ref property is also injected by XOLite, so she is not present here.
|
||||
@@ -77,6 +82,7 @@ import type { Xapi } from './lib/xen-orchestra-xapi.mjs'
|
||||
type WrapperXenApi<T, Type extends string, Fn = { (): void }> = T & {
|
||||
$call: Fn
|
||||
$callAsync: Fn
|
||||
$pool: XenApiPool
|
||||
$type: Type
|
||||
$snapshot(params: {
|
||||
cancelToken?: unknown
|
||||
@@ -162,6 +168,8 @@ export interface XenApiEvent {
|
||||
timestamp?: string
|
||||
}
|
||||
|
||||
type XenApiPoolCallMethods = TagCallMethods & {}
|
||||
|
||||
export interface XenApiPool {
|
||||
$ref: Branded<'pool'>
|
||||
allowed_operations: POOL_ALLOWED_OPERATIONS[]
|
||||
@@ -224,7 +232,7 @@ export interface XenApiPool {
|
||||
/** @deprecated */
|
||||
wlb_verify_cert?: boolean
|
||||
}
|
||||
export type XenApiPoolWrapped = WrapperXenApi<XenApiPool, 'pool'>
|
||||
export type XenApiPoolWrapped = WrapperXenApi<XenApiPool, 'pool', XenApiPoolCallMethods>
|
||||
|
||||
/** @deprecated */
|
||||
export interface XenApiPoolPatch {
|
||||
@@ -256,7 +264,7 @@ export interface XenApiPoolUpdate {
|
||||
version?: string
|
||||
}
|
||||
|
||||
type XenApiVmCallMethods = {
|
||||
type XenApiVmCallMethods = TagCallMethods & {
|
||||
(method: 'start', start_paused: boolean, force: boolean): Promise<void>
|
||||
(method: 'clean_shutdown'): Promise<void>
|
||||
(method: 'hard_shutdown'): Promise<void>
|
||||
@@ -483,6 +491,7 @@ export interface XenApiDrTask {
|
||||
uuid: string
|
||||
}
|
||||
|
||||
type XenApiHostCallMethods = TagCallMethods & {}
|
||||
export interface XenApiHost {
|
||||
$ref: Branded<'host'>
|
||||
address: string
|
||||
@@ -559,7 +568,7 @@ export interface XenApiHost {
|
||||
uuid: string
|
||||
virtual_hardware_platform_versions: number[]
|
||||
}
|
||||
export type XenApiHostWrapped = WrapperXenApi<XenApiHost, 'host'>
|
||||
export type XenApiHostWrapped = WrapperXenApi<XenApiHost, 'host', XenApiHostCallMethods>
|
||||
|
||||
export interface XenApiHostCrashdump {
|
||||
$ref: Branded<'host_crashdump'>
|
||||
@@ -614,6 +623,10 @@ export interface XenApiHostCpu {
|
||||
vendor: string
|
||||
}
|
||||
|
||||
type XenApiNetworkCallMethods = TagCallMethods & {
|
||||
(method: 'add_purpose', purpose: NETWORK_PURPOSE): Promise<void>
|
||||
}
|
||||
|
||||
export interface XenApiNetwork {
|
||||
$ref: Branded<'network'>
|
||||
allowed_operations: NETWORK_OPERATIONS[]
|
||||
@@ -633,8 +646,7 @@ export interface XenApiNetwork {
|
||||
uuid: string
|
||||
VIFs: XenApiVif['$ref'][]
|
||||
}
|
||||
export interface XenApiNetworkWrapped extends WrapperXenApi<XenApiNetwork, 'network'> {}
|
||||
|
||||
export interface XenApiNetworkWrapped extends WrapperXenApi<XenApiNetwork, 'network', XenApiNetworkCallMethods> {}
|
||||
export interface XenApiVif {
|
||||
$ref: Branded<'VIF'>
|
||||
allowed_operations: VIF_OPERATIONS[]
|
||||
@@ -782,6 +794,7 @@ export interface XenApiSm {
|
||||
}
|
||||
export type XenApiSmWrapped = WrapperXenApi<XenApiSm, 'SM'>
|
||||
|
||||
type XenApiSrCallMethods = TagCallMethods & {}
|
||||
export interface XenApiSr {
|
||||
$ref: Branded<'SR'>
|
||||
allowed_operations: STORAGE_OPERATIONS[]
|
||||
@@ -806,7 +819,7 @@ export interface XenApiSr {
|
||||
VDIs: XenApiVdi['$ref'][]
|
||||
virtual_allocation: number
|
||||
}
|
||||
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR'>
|
||||
export type XenApiSrWrapped = WrapperXenApi<XenApiSr, 'SR', XenApiSrCallMethods>
|
||||
|
||||
export interface XenApiSrStat {
|
||||
$ref: Branded<'sr_stat'>
|
||||
@@ -832,6 +845,8 @@ export interface XenApiLvhd {
|
||||
uuid: string
|
||||
}
|
||||
|
||||
type XenApiVdiCallMethods = TagCallMethods & {}
|
||||
|
||||
export interface XenApiVdi {
|
||||
$ref: Branded<'VDI'>
|
||||
allow_caching: boolean
|
||||
@@ -868,7 +883,7 @@ export interface XenApiVdi {
|
||||
virtual_size: number
|
||||
xenstore_data: Record<string, string>
|
||||
}
|
||||
export type XenApiVdiWrapped = WrapperXenApi<XenApiVdi, 'VDI'>
|
||||
export type XenApiVdiWrapped = WrapperXenApi<XenApiVdi, 'VDI', XenApiVdiCallMethods>
|
||||
|
||||
export interface XenApiVbd {
|
||||
$ref: Branded<'VBD'>
|
||||
@@ -921,6 +936,7 @@ export interface XenApiPbd {
|
||||
SR: XenApiSr['$ref']
|
||||
uuid: string
|
||||
}
|
||||
export type XenApiPbdWrapped = WrapperXenApi<XenApiPbd, 'PBD'>
|
||||
|
||||
/** @deprecated */
|
||||
export interface XenApiCrashdump {
|
||||
@@ -995,6 +1011,7 @@ export interface XenApiMessage {
|
||||
timestamp: string
|
||||
uuid: string
|
||||
}
|
||||
export type XenApiMessageWrapped = WrapperXenApi<XenApiMessage, 'message'>
|
||||
|
||||
export interface XenApiSecret {
|
||||
$ref: Branded<'secret'>
|
||||
@@ -1357,6 +1374,7 @@ export type XenApiRecord =
|
||||
export type WrappedXenApiRecord =
|
||||
| XenApiHostWrapped
|
||||
| XenApiNetworkWrapped
|
||||
| XenApiPbdWrapped
|
||||
| XenApiPifWrapped
|
||||
| XenApiPoolWrapped
|
||||
| XenApiSrWrapped
|
||||
|
||||
257
@vates/types/src/xo-app.mts
Normal file
257
@vates/types/src/xo-app.mts
Normal file
@@ -0,0 +1,257 @@
|
||||
import { EventEmitter } from 'node:stream'
|
||||
import type {
|
||||
AnyXoBackupJob,
|
||||
AnyXoJob,
|
||||
AnyXoLog,
|
||||
XapiXoRecord,
|
||||
XoAuthenticationToken,
|
||||
XoBackupRepository,
|
||||
XoConfigBackupArchive,
|
||||
XoGroup,
|
||||
XoHost,
|
||||
XoPool,
|
||||
XoPoolBackupArchive,
|
||||
XoProxy,
|
||||
XoSchedule,
|
||||
XoServer,
|
||||
XoTask,
|
||||
XoUser,
|
||||
XoVm,
|
||||
XoVmBackupArchive,
|
||||
} from './xo.mjs'
|
||||
import { VatesTask } from './lib/vates-task.mjs'
|
||||
import {
|
||||
Xapi,
|
||||
XapiHostStats,
|
||||
XapiPoolStats,
|
||||
XapiStatsGranularity,
|
||||
XapiVmStats,
|
||||
XenApiGpuGroupWrapped,
|
||||
XenApiHostWrapped,
|
||||
XenApiMessageWrapped,
|
||||
XenApiNetworkWrapped,
|
||||
XenApiPbdWrapped,
|
||||
XenApiPciWrapped,
|
||||
XenApiPgpuWrapped,
|
||||
XenApiPifWrapped,
|
||||
XenApiPoolWrapped,
|
||||
XenApiSmWrapped,
|
||||
XenApiSrWrapped,
|
||||
XenApiVbdWrapped,
|
||||
XenApiVdiWrapped,
|
||||
XenApiVgpuTypeWrapped,
|
||||
XenApiVgpuWrapped,
|
||||
XenApiVifWrapped,
|
||||
XenApiVmWrapped,
|
||||
XenApiVtpmWrapped,
|
||||
} from './index.mjs'
|
||||
|
||||
export type XapiConnection = Xapi & {
|
||||
status: string
|
||||
pool?: { uuid: string }
|
||||
sessionId: string
|
||||
_url?: { protocol: string; hostname: string; port?: string }
|
||||
}
|
||||
|
||||
type XapiRecordByXapiXoRecord = {
|
||||
gpuGroup: XenApiGpuGroupWrapped
|
||||
host: XenApiHostWrapped
|
||||
message: XenApiMessageWrapped
|
||||
network: XenApiNetworkWrapped
|
||||
PBD: XenApiPbdWrapped
|
||||
PCI: XenApiPciWrapped
|
||||
PGPU: XenApiPgpuWrapped
|
||||
PIF: XenApiPifWrapped
|
||||
pool: XenApiPoolWrapped
|
||||
SR: XenApiSrWrapped
|
||||
SM: XenApiSmWrapped
|
||||
VBD: XenApiVbdWrapped
|
||||
VDI: XenApiVdiWrapped
|
||||
'VDI-snapshot': XenApiVdiWrapped
|
||||
'VDI-unmanaged': XenApiVdiWrapped
|
||||
vgpu: XenApiVgpuWrapped
|
||||
vgpuType: XenApiVgpuTypeWrapped
|
||||
VIF: XenApiVifWrapped
|
||||
VM: XenApiVmWrapped
|
||||
'VM-controller': XenApiVmWrapped
|
||||
'VM-snapshot': XenApiVmWrapped
|
||||
'VM-template': XenApiVmWrapped
|
||||
VTPM: XenApiVtpmWrapped
|
||||
}
|
||||
export type XoApp = {
|
||||
config: {
|
||||
get<T = string>(path: string): T
|
||||
getOptional(path: string): Record<string, string> | undefined
|
||||
getOptionalDuration(path: string): number | undefined
|
||||
getGuiRoutes(): Promise<{
|
||||
default: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
v5?: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
v6?: {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
[key: string]:
|
||||
| undefined
|
||||
| {
|
||||
url: string
|
||||
path: string
|
||||
}
|
||||
}>
|
||||
}
|
||||
|
||||
objects: EventEmitter & {
|
||||
allIndexes: {
|
||||
type: {
|
||||
getEventEmitterByType(type: XapiXoRecord['type']): EventEmitter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks: EventEmitter & {
|
||||
abort(id: XoTask['id'], reason?: string): Promise<void>
|
||||
clearLogs(): Promise<void>
|
||||
create: (params: { name: string; objectId?: string; type?: string }) => VatesTask
|
||||
deleteLog(id: XoTask['id']): Promise<void>
|
||||
get(id: XoTask['id']): Promise<XoTask>
|
||||
list(opts?: { filter?: string | ((obj: XoTask) => boolean); limit?: number }): AsyncGenerator<XoTask>
|
||||
watch(id: XoTask['id'], cb: (task: XoTask) => void): Promise<() => void>
|
||||
}
|
||||
apiContext: {
|
||||
user?: XoUser
|
||||
permission?: XoUser['permission'] | 'none' | null
|
||||
}
|
||||
|
||||
// methods ------------
|
||||
addUserToGroup: (userId: XoUser['id'], groupId: XoGroup['id']) => Promise<void>
|
||||
authenticateUser: (
|
||||
credentials: { token?: string; username?: string; password?: string },
|
||||
userData?: { ip?: string },
|
||||
opts?: { bypassOtp?: boolean; bypassTaskCreation?: boolean }
|
||||
) => Promise<{ bypassOtp: boolean; expiration: number; user: XoUser }>
|
||||
/* Throw if no authorization */
|
||||
checkFeatureAuthorization(featureCode: string): Promise<void>
|
||||
/* connect a server (XCP-ng/XenServer) */
|
||||
connectXenServer(id: XoServer['id']): Promise<void>
|
||||
createAuthenticationToken(opts: {
|
||||
client?: {
|
||||
id?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
description?: string
|
||||
expiresIn?: string | number
|
||||
userId: XoUser['id']
|
||||
}): Promise<XoAuthenticationToken>
|
||||
createUser(params: { name?: string; password?: string; [key: string]: unknown }): Promise<XoUser>
|
||||
deleteGroup(id: XoGroup['id']): Promise<void>
|
||||
deleteUser(id: XoUser['id']): Promise<void>
|
||||
createGroup(params: { name: string; provider?: string; providerGroup?: string }): Promise<XoGroup>
|
||||
/* disconnect a server (XCP-ng/XenServer) */
|
||||
disconnectXenServer(id: XoServer['id']): Promise<void>
|
||||
getAllGroups(): Promise<XoGroup[]>
|
||||
getAllProxies(): Promise<XoProxy[]>
|
||||
getAllJobs<T extends AnyXoBackupJob['type']>(type: T): Promise<Extract<AnyXoBackupJob, { type: T }>[]>
|
||||
getAllJobs(type?: string): Promise<AnyXoJob[]>
|
||||
getProxy(id: XoProxy['id']): Promise<XoProxy>
|
||||
getRemote(id: XoBackupRepository['id']): Promise<XoBackupRepository>
|
||||
getAllRemotes(): Promise<XoBackupRepository[]>
|
||||
getAllRemotesInfo(): Promise<
|
||||
Record<
|
||||
XoBackupRepository['id'],
|
||||
{
|
||||
size?: number
|
||||
used?: number
|
||||
available?: number
|
||||
encryption?: {
|
||||
algorithm: string
|
||||
isLegacy: boolean
|
||||
recommanded: string
|
||||
}
|
||||
}
|
||||
>
|
||||
>
|
||||
getAllSchedules(): Promise<XoSchedule[]>
|
||||
getAllUsers(): Promise<XoUser[]>
|
||||
getAllXenServers(): Promise<XoServer[]>
|
||||
getAuthenticationTokensForUser(userId: XoUser['id']): Promise<XoAuthenticationToken[]>
|
||||
getBackupNgLogs(): Promise<Record<string, AnyXoLog>>
|
||||
getBackupNgLogs(id: AnyXoLog['id']): Promise<AnyXoLog>
|
||||
getBackupNgLogsSorted(opts: {
|
||||
after?: number
|
||||
before?: number
|
||||
filter: (log: AnyXoLog) => boolean
|
||||
limit?: number
|
||||
}): Promise<AnyXoLog[]>
|
||||
getGroup(id: XoGroup['id']): Promise<XoGroup>
|
||||
getHVSupportedVersions: undefined | (() => Promise<{ [key: XoHost['productBrand']]: string }>)
|
||||
getJob<T extends AnyXoJob>(id: T['id']): Promise<T>
|
||||
getObject: <T extends XapiXoRecord>(id: T['id'], type?: T['type'] | T['type'][]) => T
|
||||
getObjectsByType: <T extends XapiXoRecord>(
|
||||
type: T['type'],
|
||||
opts?: { filter?: string | ((obj: T) => boolean); limit?: number }
|
||||
) => Record<T['id'], T> | undefined
|
||||
getTotalBackupSizeOnRemote(id: XoBackupRepository['id']): Promise<{ onDisk: number }>
|
||||
getSchedule(id: XoSchedule['id']): Promise<XoSchedule>
|
||||
getUser: (id: XoUser['id']) => Promise<XoUser>
|
||||
getXapi(maybeId: XapiXoRecord['id'] | XapiXoRecord): Xapi
|
||||
getXapiHostStats: (hostId: XoHost['id'], granularity?: XapiStatsGranularity) => Promise<XapiHostStats>
|
||||
getXapiObject: <T extends XapiXoRecord>(
|
||||
maybeId: T['id'] | T,
|
||||
type: T['type'] | T['type'][]
|
||||
) => XapiRecordByXapiXoRecord[T['type']]
|
||||
getXapiPoolStats(poolId: XoPool['id'], granularity?: XapiStatsGranularity): Promise<XapiPoolStats>
|
||||
getXapiVmStats: (vmId: XoVm['id'], granularity?: XapiStatsGranularity) => Promise<XapiVmStats>
|
||||
getXenServer(id: XoServer['id']): Promise<XoServer>
|
||||
hasFeatureAuthorization(featureCode: string): Promise<boolean>
|
||||
hasObject<T extends XapiXoRecord>(id: T['id'], type: T['type']): boolean
|
||||
listMetadataBackups(backupRepositoryIds: XoBackupRepository['id'][]): Promise<{
|
||||
xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
|
||||
pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
|
||||
}>
|
||||
listVmBackupsNg(
|
||||
backupRepositoryIds: XoBackupRepository['id'][],
|
||||
opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
|
||||
): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
|
||||
/** Allow to add a new server in the DB (XCP-ng/XenServer) */
|
||||
registerXenServer(
|
||||
body: Pick<XoServer, 'host' | 'httpProxy' | 'label' | 'username'> & {
|
||||
allowUnauthorized?: XoServer['allowUnauthorized']
|
||||
password: string
|
||||
readOnly?: XoServer['readOnly']
|
||||
}
|
||||
): Promise<XoServer>
|
||||
rollingPoolReboot(pool: XoPool, opts?: { parentTask?: VatesTask }): Promise<void>
|
||||
rollingPoolUpdate(pool: XoPool, opts?: { rebootVm?: boolean; parentTask?: VatesTask }): Promise<void>
|
||||
removeUserFromGroup(userId: XoUser['id'], id: XoGroup['id']): Promise<void>
|
||||
runJob(job: AnyXoJob, schedule: XoSchedule): void
|
||||
runWithApiContext: (user: XoUser | undefined, fn: () => void) => Promise<unknown>
|
||||
/** Remove a server from the DB (XCP-ng/XenServer) */
|
||||
unregisterXenServer(id: XoServer['id']): Promise<void>
|
||||
updateUser(
|
||||
id: XoUser['id'],
|
||||
updates: {
|
||||
/**
|
||||
* @deprecated
|
||||
*/
|
||||
email?: string
|
||||
authProviders?: Record<string, string>
|
||||
name?: string
|
||||
password?: string
|
||||
permission?: string
|
||||
preferences?: Record<string, string>
|
||||
}
|
||||
): Promise<void>
|
||||
updateGroup(
|
||||
id: XoGroup['id'],
|
||||
updates: {
|
||||
name?: string
|
||||
}
|
||||
): void
|
||||
getAllXapis(): Record<string, XapiConnection>
|
||||
getObjects(opts?: { filter?: Record<string, unknown>; limit?: number }): Record<string, XapiXoRecord>
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
VM_POWER_STATE,
|
||||
} from './common.mjs'
|
||||
import type * as CMType from './lib/complex-matcher.mjs'
|
||||
import type { XenApiHost, XenApiPool } from './xen-api.mjs'
|
||||
|
||||
type BaseXapiXo = {
|
||||
$pool: XoPool['id']
|
||||
@@ -92,7 +93,7 @@ type BaseXoVm = BaseXapiXo & {
|
||||
notes?: string
|
||||
os_version: Record<string, string> | null
|
||||
other: Record<string, string>
|
||||
parent?: XoVm['id']
|
||||
parent?: XoVmSnapshot['id']
|
||||
power_state: VM_POWER_STATE
|
||||
pvDriversDetected?: boolean
|
||||
pvDriversUpToDate?: boolean
|
||||
@@ -105,6 +106,10 @@ type BaseXoVm = BaseXapiXo & {
|
||||
suspendSr?: XoSr['id']
|
||||
tags: string[]
|
||||
vga?: string
|
||||
/**
|
||||
* Value in MiB.
|
||||
* See: https://wiki.xenproject.org/wiki/XCP_PV_templates_start
|
||||
*/
|
||||
videoram?: number
|
||||
viridian: boolean
|
||||
virtualizationMode: DOMAIN_TYPE
|
||||
@@ -136,14 +141,63 @@ export type XoAlarm = Omit<XoMessage, '$object' | 'body'> & {
|
||||
// TODO: to be typed when Bastien.N has finished working on the XO task
|
||||
type BaseXoLog = {
|
||||
id: Branded<'xo-log'>
|
||||
infos?: { data: unknown; message: string }[]
|
||||
[key: string]: unknown
|
||||
}
|
||||
export type XoBackupLog = BaseXoLog & {
|
||||
message: 'backup' | 'metadata'
|
||||
status: 'success' | 'skipped' | 'interrupted' | 'failure' | 'pending'
|
||||
message: 'backup'
|
||||
start: number
|
||||
end?: number
|
||||
tasks?: XoTask[]
|
||||
jobId?: AnyXoBackupJob['id']
|
||||
}
|
||||
export type XoRestoreLog = BaseXoLog & {
|
||||
message: 'restore'
|
||||
}
|
||||
export type XoVmBackupArchive = {
|
||||
id: Branded<'vm-backup-archive'>
|
||||
type: 'xo-vm-backup'
|
||||
backupRepository: XoBackupRepository['id']
|
||||
disks: { id: string; name: string; uuid: XoVdiSnapshot['uuid'] }[]
|
||||
isImmutable?: boolean
|
||||
jobId: XoVmBackupJob['id']
|
||||
mode: XoVmBackupJob['mode']
|
||||
scheduleId: XoSchedule['id']
|
||||
size: number
|
||||
timestamp: number
|
||||
vm: {
|
||||
uuid: XoVm['uuid']
|
||||
name_description: string
|
||||
name_label: string
|
||||
tags: XoVm['tags']
|
||||
}
|
||||
differencingVhds?: number
|
||||
dynamicVhds?: number
|
||||
withMemory: boolean
|
||||
}
|
||||
|
||||
type XoMetadataBackupArchive = {
|
||||
id: Branded<'metadata-backup-archive'>
|
||||
backupRepository: XoBackupRepository['id']
|
||||
jobId: XoVmBackupJob['id']
|
||||
jobName: XoVmBackupJob['name']
|
||||
scheduleId: XoSchedule['id']
|
||||
scheduleName: XoSchedule['name']
|
||||
timestamp: number
|
||||
}
|
||||
export type XoConfigBackupArchive = XoMetadataBackupArchive & {
|
||||
id: Branded<'config-backup-archive'>
|
||||
data: string
|
||||
type: 'xo-config-backup'
|
||||
}
|
||||
|
||||
export type XoPoolBackupArchive = XoMetadataBackupArchive & {
|
||||
id: Branded<'pool-backup-archive'>
|
||||
pool: XenApiPool
|
||||
poolMaster: XenApiHost
|
||||
type: 'xo-pool-metadata-backup'
|
||||
}
|
||||
|
||||
export type XoBackupRepository = {
|
||||
benchmarks?: { readRate: number; timestamp: number; writeRate: number }[]
|
||||
@@ -185,7 +239,7 @@ export type XoHost = BaseXapiXo & {
|
||||
|
||||
address: string
|
||||
agentStartTime: null | number
|
||||
bios_string: Record<string, string>
|
||||
bios_strings: Record<string, string>
|
||||
build: string
|
||||
certificates?: {
|
||||
fingerprint: string
|
||||
@@ -194,7 +248,7 @@ export type XoHost = BaseXapiXo & {
|
||||
chipset_info: {
|
||||
iommu?: boolean
|
||||
}
|
||||
controlDomain?: XoVm['id']
|
||||
controlDomain?: XoVmController['id']
|
||||
cpus: {
|
||||
cores?: number
|
||||
sockets?: number
|
||||
@@ -289,7 +343,19 @@ export type XoNetwork = BaseXapiXo & {
|
||||
}
|
||||
|
||||
export type XoPbd = BaseXapiXo & {
|
||||
attached: boolean
|
||||
device_config:
|
||||
| { device: string }
|
||||
| { location: string }
|
||||
| { path: string; location: string; legacy_mode: string }
|
||||
| { provisioning: string; redundancy: string; 'group-name': string }
|
||||
| { server: string; serverpath: string }
|
||||
| { type: string; location: string }
|
||||
| Record<string, string>
|
||||
host: XoHost['id']
|
||||
id: Branded<'PBD'>
|
||||
otherConfig: Record<string, string>
|
||||
SR: XoSr['id']
|
||||
type: 'PBD'
|
||||
}
|
||||
|
||||
@@ -330,7 +396,7 @@ export type XoPif = BaseXapiXo & {
|
||||
|
||||
attached: boolean
|
||||
bondMaster?: XoPif['id']
|
||||
bondSalves?: XoPif['id'][]
|
||||
bondSlaves?: XoPif['id'][]
|
||||
carrier: boolean
|
||||
device: string
|
||||
deviceName?: string
|
||||
@@ -363,7 +429,7 @@ export type XoPool = BaseXapiXo & {
|
||||
}
|
||||
crashDumpSr?: XoSr['id']
|
||||
current_operations: Record<string, POOL_ALLOWED_OPERATIONS>
|
||||
defaultSr?: XoSr['id']
|
||||
default_SR?: XoSr['id']
|
||||
HA_enabled: boolean
|
||||
haSrs: XoSr['id'][]
|
||||
id: Branded<'pool'>
|
||||
@@ -377,7 +443,6 @@ export type XoPool = BaseXapiXo & {
|
||||
tags: string[]
|
||||
type: 'pool'
|
||||
vtpmSupported: boolean
|
||||
xosanPackInstallationTime: number | null
|
||||
zstdSupported: boolean
|
||||
}
|
||||
|
||||
@@ -400,6 +465,7 @@ type BaseXoJob = {
|
||||
type XoBackupJobGeneralSettings = {
|
||||
backupReportTpl?: 'compactMjml' | 'mjml'
|
||||
reportWhen?: 'always' | 'error' | 'failure' | 'never'
|
||||
reportRecipients?: string[]
|
||||
hideSuccessfulItems?: boolean
|
||||
[key: string]: unknown
|
||||
}
|
||||
@@ -413,6 +479,8 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
monthly?: { retention: number; settings: Record<string, unknown> }
|
||||
yearly?: { retention: number; settings: Record<string, unknown> }
|
||||
}
|
||||
checkpointSnapshot?: boolean
|
||||
offlineSnapshot?: boolean
|
||||
maxExportRate?: number
|
||||
nbdConcurrency?: number
|
||||
nRetriesVmBackupFailures?: number
|
||||
@@ -421,7 +489,6 @@ export type XoVmBackupJobGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
mergeBackupsSynchronously?: boolean
|
||||
offlineBackup?: boolean
|
||||
timeout?: number
|
||||
[key: string]: unknown
|
||||
}
|
||||
export type XoVmBackupJobScheduleSettings = {
|
||||
exportRetention?: number
|
||||
@@ -469,11 +536,11 @@ export type XoMetadataBackupJob = BaseXoJob & {
|
||||
export type XoMirrorBackupGeneralSettings = XoBackupJobGeneralSettings & {
|
||||
concurrency?: number
|
||||
nRetriesVmBackupFailures?: number
|
||||
mergeBackupsSynchronously?: boolean
|
||||
timeout?: number
|
||||
maxExportRate?: number
|
||||
backupReportTpl?: 'compactMjml'
|
||||
reportWhen: 'failure'
|
||||
[key: string]: unknown
|
||||
}
|
||||
export type XoMirrorBackupScheduleSettings = {
|
||||
exportRetention?: number
|
||||
@@ -486,13 +553,16 @@ export type XoMirrorBackupJob = BaseXoJob & {
|
||||
mode: 'full' | 'delta'
|
||||
sourceRemote: XoBackupRepository['id']
|
||||
remotes: CMType.IdOr<XoBackupRepository['id']>
|
||||
proxy?: XoProxy['id']
|
||||
settings: {
|
||||
'': XoMirrorBackupGeneralSettings
|
||||
[scheduleId: XoSchedule['id']]: XoMirrorBackupScheduleSettings | undefined
|
||||
}
|
||||
}
|
||||
|
||||
export type XoJob = BaseXoJob & {}
|
||||
export type XoJob = BaseXoJob & {
|
||||
type: 'call'
|
||||
}
|
||||
|
||||
export type XoSchedule = {
|
||||
cron: string
|
||||
@@ -527,7 +597,7 @@ export type XoSr = BaseXapiXo & {
|
||||
|
||||
VDIs: AnyXoVdi['id'][]
|
||||
|
||||
allocationStrategy: 'thin' | 'thick' | 'unknown'
|
||||
allocationStrategy?: 'thin' | 'thick' | 'unknown'
|
||||
content_type: string
|
||||
current_operations: Record<string, STORAGE_OPERATIONS>
|
||||
id: Branded<'SR'>
|
||||
@@ -562,14 +632,32 @@ export type XoSm = BaseXapiXo & {
|
||||
|
||||
export type XoTask = {
|
||||
abortionRequestedAt?: number
|
||||
data?: Record<string, string>
|
||||
end?: number
|
||||
id: Branded<'task'>
|
||||
infos?: { data: unknown; message: string }[]
|
||||
progress?: number
|
||||
properties: {
|
||||
method?: string
|
||||
name?: string
|
||||
objectId?: string
|
||||
objectType?:
|
||||
| XapiXoRecord['type']
|
||||
| 'backup'
|
||||
| 'backup-archive'
|
||||
| 'backup-job'
|
||||
| 'backup-log'
|
||||
| 'backup-repository'
|
||||
| 'group'
|
||||
| 'proxy'
|
||||
| 'restore'
|
||||
| 'restore-log'
|
||||
| 'schedule'
|
||||
| 'server'
|
||||
| 'task'
|
||||
| 'user'
|
||||
params?: Record<string, unknown>
|
||||
progress?: number
|
||||
type?: string
|
||||
userId?: string
|
||||
[key: string]: unknown | undefined
|
||||
@@ -641,6 +729,7 @@ type BaseXoVdi = BaseXapiXo & {
|
||||
|
||||
export type XoVdi = BaseXoVdi & {
|
||||
id: Branded<'VDI'>
|
||||
chainPhysicalUsage: number
|
||||
type: 'VDI'
|
||||
}
|
||||
|
||||
@@ -741,6 +830,7 @@ export type XapiXoRecord =
|
||||
| XoHost
|
||||
| XoMessage
|
||||
| XoNetwork
|
||||
| XoPbd
|
||||
| XoPci
|
||||
| XoPgpu
|
||||
| XoPif
|
||||
@@ -761,10 +851,14 @@ export type XapiXoRecord =
|
||||
| XoSm
|
||||
|
||||
export type NonXapiXoRecord =
|
||||
| AnyXoBackupArchive
|
||||
| AnyXoJob
|
||||
| AnyXoLog
|
||||
| XoGroup
|
||||
| XoProxy
|
||||
| XoGroup
|
||||
| XoProxy
|
||||
| XoJob
|
||||
| XoBackupRepository
|
||||
| XoSchedule
|
||||
| XoServer
|
||||
@@ -782,3 +876,5 @@ export type AnyXoJob = XoJob | AnyXoBackupJob
|
||||
export type AnyXoBackupJob = XoVmBackupJob | XoMetadataBackupJob | XoMirrorBackupJob
|
||||
|
||||
export type AnyXoLog = XoBackupLog | XoRestoreLog
|
||||
|
||||
export type AnyXoBackupArchive = XoVmBackupArchive | XoConfigBackupArchive | XoPoolBackupArchive
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
"strict": true,
|
||||
"module": "ESNext",
|
||||
"target": "ES2022",
|
||||
"skipLibCheck": true
|
||||
"skipLibCheck": true,
|
||||
"types": ["node"] // Only use types from Node.js and packages that are explicitly imported
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/async-map",
|
||||
"version": "0.1.2",
|
||||
"version": "0.1.3",
|
||||
"license": "ISC",
|
||||
"description": "Promise.all + map for all iterables",
|
||||
"keywords": [
|
||||
@@ -28,7 +28,7 @@
|
||||
"node": ">=6"
|
||||
},
|
||||
"dependencies": {
|
||||
"lodash": "^4.17.4"
|
||||
"lodash": "^4.18.0"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish",
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/decorate-with": "^2.1.0",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"golike-defer": "^0.5.1",
|
||||
"object-hash": "^2.0.1"
|
||||
},
|
||||
|
||||
@@ -6,13 +6,13 @@
|
||||
"preferGlobal": true,
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/backups": "^0.64.3",
|
||||
"@xen-orchestra/fs": "^4.6.3",
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/backups": "^0.72.0",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"filenamify": "^6.0.0",
|
||||
"getopts": "^2.2.5",
|
||||
"lodash": "^4.17.15",
|
||||
"lodash": "^4.18.0",
|
||||
"promise-toolbox": "^0.21.0"
|
||||
},
|
||||
"engines": {
|
||||
@@ -28,7 +28,7 @@
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"version": "1.1.3",
|
||||
"version": "1.1.11",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Task } from './Task.mjs'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
export class HealthCheckVmBackup {
|
||||
#restoredVm
|
||||
@@ -14,7 +14,7 @@ export class HealthCheckVmBackup {
|
||||
async run() {
|
||||
return Task.run(
|
||||
{
|
||||
name: 'vmstart',
|
||||
properties: { name: 'vmstart' },
|
||||
},
|
||||
async () => {
|
||||
let restoredVm = this.#restoredVm
|
||||
|
||||
@@ -2,15 +2,16 @@ import assert from 'node:assert'
|
||||
|
||||
import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { importIncrementalVm } from './_incrementalVm.mjs'
|
||||
import { Task } from './Task.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
import { decorateClass } from '@vates/decorate-with'
|
||||
import { Task } from '@vates/task'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { dirname, join } from 'node:path'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
import { defer } from 'golike-defer'
|
||||
import { NegativeDisk } from '@xen-orchestra/disk-transform'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { resetVmOtherConfig } from './_otherConfig.mjs'
|
||||
|
||||
const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
|
||||
async function resolveUuid(xapi, cache, uuid, type) {
|
||||
@@ -29,10 +30,10 @@ export class ImportVmBackup {
|
||||
metadata,
|
||||
srUuid,
|
||||
xapi,
|
||||
settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
|
||||
settings: { additionalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
|
||||
}) {
|
||||
this._adapter = adapter
|
||||
this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
|
||||
this._importIncrementalVmSettings = { additionalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
|
||||
this._metadata = metadata
|
||||
this._srUuid = srUuid
|
||||
this._xapi = xapi
|
||||
@@ -61,11 +62,11 @@ export class ImportVmBackup {
|
||||
const { mapVdisSrs } = this._importIncrementalVmSettings
|
||||
const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
|
||||
const disks = {}
|
||||
const metdataDir = dirname(metadata._filename)
|
||||
const metadataDir = dirname(metadata._filename)
|
||||
const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
|
||||
|
||||
for (const [vdiRef, vdi] of Object.entries(vdis)) {
|
||||
const vhdPath = join(metdataDir, vhds[vdiRef])
|
||||
const vhdPath = join(metadataDir, vhds[vdiRef])
|
||||
|
||||
let xapiDisk
|
||||
try {
|
||||
@@ -77,7 +78,7 @@ export class ImportVmBackup {
|
||||
|
||||
let snapshotCandidate, backupCandidate
|
||||
if (xapiDisk !== undefined) {
|
||||
debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
|
||||
debug('found disks, will search its snapshots', { snapshots: xapiDisk.snapshots })
|
||||
for (const snapshotRef of xapiDisk.snapshots) {
|
||||
const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
|
||||
|
||||
@@ -96,7 +97,7 @@ export class ImportVmBackup {
|
||||
// have a corresponding backup more recent than metadata ?
|
||||
const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
|
||||
if (pathToSnapshotData === undefined) {
|
||||
debug('no backup linked to this snaphot')
|
||||
debug('no backup linked to this snapshot')
|
||||
continue
|
||||
}
|
||||
if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
|
||||
@@ -112,7 +113,7 @@ export class ImportVmBackup {
|
||||
}
|
||||
|
||||
let disk
|
||||
const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
|
||||
const backupWithSnapshotPath = join(metadataDir, backupCandidate ?? '')
|
||||
if (vhdPath === backupWithSnapshotPath) {
|
||||
// all the data are already on the host
|
||||
debug('direct reuse of a snapshot')
|
||||
@@ -194,7 +195,7 @@ export class ImportVmBackup {
|
||||
}
|
||||
|
||||
async #decorateIncrementalVmMetadata() {
|
||||
const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
|
||||
const { additionalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
|
||||
|
||||
const ignoredVdis = new Set(
|
||||
Object.entries(mapVdisSrs)
|
||||
@@ -211,8 +212,8 @@ export class ImportVmBackup {
|
||||
|
||||
const cache = new Map()
|
||||
const mapVdisSrRefs = {}
|
||||
if (additionnalVmTag !== undefined) {
|
||||
backup.vm.tags.push(additionnalVmTag)
|
||||
if (additionalVmTag !== undefined) {
|
||||
backup.vm.tags.push(additionalVmTag)
|
||||
}
|
||||
for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
|
||||
mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
|
||||
@@ -243,7 +244,7 @@ export class ImportVmBackup {
|
||||
|
||||
return Task.run(
|
||||
{
|
||||
name: 'transfer',
|
||||
properties: { name: 'transfer' },
|
||||
},
|
||||
async () => {
|
||||
const xapi = this._xapi
|
||||
@@ -254,6 +255,14 @@ export class ImportVmBackup {
|
||||
: await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
|
||||
newMacAddresses,
|
||||
})
|
||||
let size = 0
|
||||
if (isFull) {
|
||||
size = sizeContainer.size
|
||||
} else {
|
||||
for (const disk of Object.values(backup.disks)) {
|
||||
size += disk.getNbGeneratedBlock() * disk.getBlockSize()
|
||||
}
|
||||
}
|
||||
const remoteName = adapter._handler._remote.name
|
||||
let desc = `Restored on ${formatFilenameDate(+new Date())}`
|
||||
if (remoteName !== undefined) {
|
||||
@@ -270,10 +279,11 @@ export class ImportVmBackup {
|
||||
`${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
|
||||
),
|
||||
xapi.call('VM.set_name_description', vmRef, desc),
|
||||
resetVmOtherConfig(xapi, vmRef),
|
||||
])
|
||||
|
||||
return {
|
||||
size: sizeContainer.size,
|
||||
size,
|
||||
id: await xapi.getField('VM', vmRef, 'uuid'),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import groupBy from 'lodash/groupBy.js'
|
||||
import pDefer from 'promise-toolbox/defer'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
import reduce from 'lodash/reduce.js'
|
||||
import tar from 'tar'
|
||||
import * as tar from 'tar'
|
||||
import zlib from 'zlib'
|
||||
|
||||
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
|
||||
@@ -27,11 +27,11 @@ import { formatFilenameDate } from './_filenameDate.mjs'
|
||||
import { getTmpDir } from './_getTmpDir.mjs'
|
||||
import { isMetadataFile } from './_backupType.mjs'
|
||||
import { isValidXva } from './_isValidXva.mjs'
|
||||
import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
|
||||
import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
|
||||
import { lvs, pvs } from './_lvm.mjs'
|
||||
import { watchStreamSize } from './_watchStreamSize.mjs'
|
||||
|
||||
import { RemoteVhd } from './disks/RemoteVhd.mjs'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { openDiskChain } from './disks/openDiskChain.mjs'
|
||||
import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { ReadAhead } from '@xen-orchestra/disk-transform'
|
||||
@@ -40,11 +40,11 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
|
||||
|
||||
export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
|
||||
|
||||
const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
|
||||
const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
|
||||
|
||||
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
|
||||
|
||||
const compareTimestamp = (a, b) => a.timestamp - b.timestamp
|
||||
export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -312,8 +312,8 @@ export class RemoteAdapter {
|
||||
}
|
||||
|
||||
async deleteVmBackups(files) {
|
||||
const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
|
||||
const { delta, full, ...others } = groupBy(metadatas, 'mode')
|
||||
const metadata = await asyncMap(files, file => this.readVmBackupMetadata(file))
|
||||
const { delta, full, ...others } = groupBy(metadata, 'mode')
|
||||
|
||||
const unsupportedModes = Object.keys(others)
|
||||
if (unsupportedModes.length !== 0) {
|
||||
@@ -498,7 +498,7 @@ export class RemoteAdapter {
|
||||
|
||||
const results = []
|
||||
await asyncMapSettled(partitions, partition =>
|
||||
partition.type === LVM_PARTITION_TYPE
|
||||
partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
|
||||
? this._listLvmLogicalVolumes(devicePath, partition, results)
|
||||
: results.push(partition)
|
||||
)
|
||||
@@ -575,7 +575,7 @@ export class RemoteAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
async #getCachabledDataListVmBackups(dir) {
|
||||
async #getCacheableDataListVmBackups(dir) {
|
||||
debug('generating cache', { path: dir })
|
||||
|
||||
const handler = this._handler
|
||||
@@ -613,6 +613,11 @@ export class RemoteAdapter {
|
||||
// if cache is missing or broken => regenerate it and return
|
||||
|
||||
async _readCacheListVmBackups(vmUuid) {
|
||||
// immutable remote can't use any caching
|
||||
// since the cache file may be non modifiable
|
||||
if (this._handler.isImmutable()) {
|
||||
return this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
|
||||
}
|
||||
const path = this.#getVmBackupsCache(vmUuid)
|
||||
|
||||
const cache = await this._readCache(path)
|
||||
@@ -622,7 +627,7 @@ export class RemoteAdapter {
|
||||
}
|
||||
|
||||
// nothing cached, or cache unreadable => regenerate it
|
||||
const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
|
||||
const backups = await this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
|
||||
if (backups === undefined) {
|
||||
return
|
||||
}
|
||||
@@ -709,7 +714,7 @@ export class RemoteAdapter {
|
||||
})
|
||||
} else {
|
||||
const stream = await toVhdStream(disk)
|
||||
await this.outputStream(path, stream, { validator })
|
||||
await this.outputStream(path, stream, { validator, checksum: false })
|
||||
await validator(path)
|
||||
}
|
||||
}
|
||||
@@ -739,7 +744,7 @@ export class RemoteAdapter {
|
||||
if (useChain) {
|
||||
disk = await openDiskChain({ handler, path })
|
||||
} else {
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
}
|
||||
disk = new ReadAhead(disk)
|
||||
@@ -779,7 +784,7 @@ export class RemoteAdapter {
|
||||
// if the remote is immutable, check if this metadata is also immutable
|
||||
try {
|
||||
// this file is not encrypted
|
||||
await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
|
||||
await this._handler._readFile(IMMUTABILITY_METADATA_FILENAME)
|
||||
remoteIsImmutable = true
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
import CancelToken from 'promise-toolbox/CancelToken'
|
||||
import Zone from 'node-zone'
|
||||
|
||||
const logAfterEnd = log => {
|
||||
const error = new Error('task has already ended')
|
||||
error.log = log
|
||||
throw error
|
||||
}
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
|
||||
|
||||
// Create a serializable object from an error.
|
||||
//
|
||||
// Otherwise some fields might be non-enumerable and missing from logs.
|
||||
const serializeError = error =>
|
||||
error instanceof Error
|
||||
? {
|
||||
...error, // Copy enumerable properties.
|
||||
code: error.code,
|
||||
errors: serializeErrors(error.errors), // supports AggregateError
|
||||
message: error.message,
|
||||
name: error.name,
|
||||
stack: error.stack,
|
||||
}
|
||||
: error
|
||||
|
||||
const $$task = Symbol('@xen-orchestra/backups/Task')
|
||||
|
||||
export class Task {
|
||||
static get cancelToken() {
|
||||
const task = Zone.current.data[$$task]
|
||||
return task !== undefined ? task.#cancelToken : CancelToken.none
|
||||
}
|
||||
|
||||
static run(opts, fn) {
|
||||
return new this(opts).run(fn, true)
|
||||
}
|
||||
|
||||
static wrapFn(opts, fn) {
|
||||
// compatibility with @decorateWith
|
||||
if (typeof fn !== 'function') {
|
||||
;[fn, opts] = [opts, fn]
|
||||
}
|
||||
|
||||
return function () {
|
||||
return Task.run(typeof opts === 'function' ? opts.apply(this, arguments) : opts, () => fn.apply(this, arguments))
|
||||
}
|
||||
}
|
||||
|
||||
#cancelToken
|
||||
#id = Math.random().toString(36).slice(2)
|
||||
#onLog
|
||||
#zone
|
||||
|
||||
constructor({ name, data, onLog }) {
|
||||
let parentCancelToken, parentId
|
||||
if (onLog === undefined) {
|
||||
const parent = Zone.current.data[$$task]
|
||||
if (parent === undefined) {
|
||||
onLog = noop
|
||||
} else {
|
||||
onLog = log => parent.#onLog(log)
|
||||
parentCancelToken = parent.#cancelToken
|
||||
parentId = parent.#id
|
||||
}
|
||||
}
|
||||
|
||||
const zone = Zone.current.fork('@xen-orchestra/backups/Task')
|
||||
zone.data[$$task] = this
|
||||
this.#zone = zone
|
||||
|
||||
const { cancel, token } = CancelToken.source(parentCancelToken && [parentCancelToken])
|
||||
this.#cancelToken = token
|
||||
this.cancel = cancel
|
||||
|
||||
this.#onLog = onLog
|
||||
|
||||
this.#log('start', {
|
||||
data,
|
||||
message: name,
|
||||
parentId,
|
||||
})
|
||||
}
|
||||
|
||||
failure(error) {
|
||||
this.#end('failure', serializeError(error))
|
||||
}
|
||||
|
||||
info(message, data) {
|
||||
this.#log('info', { data, message })
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a function in the context of this task
|
||||
*
|
||||
* In case of error, the task will be failed.
|
||||
*
|
||||
* @typedef Result
|
||||
* @param {() => Result} fn
|
||||
* @param {boolean} last - Whether the task should succeed if there is no error
|
||||
* @returns Result
|
||||
*/
|
||||
run(fn, last = false) {
|
||||
return this.#zone.run(() => {
|
||||
try {
|
||||
const result = fn()
|
||||
let then
|
||||
if (result != null && typeof (then = result.then) === 'function') {
|
||||
then.call(result, last && (value => this.success(value)), error => this.failure(error))
|
||||
} else if (last) {
|
||||
this.success(result)
|
||||
}
|
||||
return result
|
||||
} catch (error) {
|
||||
this.failure(error)
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
success(value) {
|
||||
this.#end('success', value)
|
||||
}
|
||||
|
||||
warning(message, data) {
|
||||
this.#log('warning', { data, message })
|
||||
}
|
||||
|
||||
wrapFn(fn, last) {
|
||||
const task = this
|
||||
return function () {
|
||||
return task.run(() => fn.apply(this, arguments), last)
|
||||
}
|
||||
}
|
||||
|
||||
#end(status, result) {
|
||||
this.#log('end', { result, status })
|
||||
this.#onLog = logAfterEnd
|
||||
}
|
||||
|
||||
#log(event, props) {
|
||||
this.#onLog({
|
||||
...props,
|
||||
event,
|
||||
taskId: this.#id,
|
||||
timestamp: Date.now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const method of ['info', 'warning']) {
|
||||
Task[method] = (...args) => Zone.current.data[$$task]?.[method](...args)
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
export const isMetadataFile = filename => filename.endsWith('.json')
|
||||
export const isVhdFile = filename => filename.endsWith('.vhd')
|
||||
export const isVhdSumFile = filename => filename.endsWith('.vhd.checksum')
|
||||
export const isXvaFile = filename => filename.endsWith('.xva')
|
||||
export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')
|
||||
|
||||
@@ -14,10 +14,10 @@ import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { deduped } from '@vates/disposable/deduped.js'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { parseDuration } from '@vates/parse-duration'
|
||||
import { Task } from '@vates/task'
|
||||
import { Xapi } from '@xen-orchestra/xapi'
|
||||
|
||||
import { RemoteAdapter } from './RemoteAdapter.mjs'
|
||||
import { Task } from './Task.mjs'
|
||||
|
||||
createCachedLookup().patchGlobal()
|
||||
|
||||
@@ -178,8 +178,8 @@ process.on('message', async message => {
|
||||
const result = message.runWithLogs
|
||||
? await Task.run(
|
||||
{
|
||||
name: 'backup run',
|
||||
onLog: data =>
|
||||
properties: { name: 'backup run', ...message.data.jobData },
|
||||
onProgress: data =>
|
||||
emitMessage({
|
||||
data,
|
||||
type: 'log',
|
||||
|
||||
@@ -83,12 +83,12 @@ test('It remove broken vhd', async () => {
|
||||
|
||||
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
|
||||
assert.equal((await handler.list(basePath)).length, 1)
|
||||
let loggued = ''
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
loggued += message
|
||||
logged += message
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: false, logInfo, logWarn: logInfo, lock: false })
|
||||
assert.equal(loggued, `VHD check error`)
|
||||
assert.equal(logged, `VHD check error`)
|
||||
// not removed
|
||||
assert.deepEqual(await handler.list(basePath), ['notReallyAVhd.vhd'])
|
||||
// really remove it
|
||||
@@ -98,7 +98,7 @@ test('It remove broken vhd', async () => {
|
||||
|
||||
test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
// one with a broken parent, should be deleted
|
||||
await generateVhd(`${basePath}/abandonned.vhd`, {
|
||||
await generateVhd(`${basePath}/abandoned.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'gone.vhd',
|
||||
parentUuid: uniqueIdBuffer(),
|
||||
@@ -118,18 +118,18 @@ test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`],
|
||||
vhds: [`${basePath}/child.vhd`, `${basePath}/abandoned.vhd`],
|
||||
}),
|
||||
{ flags: 'w' }
|
||||
)
|
||||
// clean
|
||||
let loggued = ''
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
loggued += message + '\n'
|
||||
logged += message + '\n'
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
|
||||
|
||||
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
|
||||
const deletedOrphanVhd = logged.match(/deleting orphan VHD/g) || []
|
||||
assert.equal(deletedOrphanVhd.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// we don't test the file on disk, since they will all be marker as unused and deleted without a metadata.json file
|
||||
@@ -144,12 +144,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
|
||||
vhds: [
|
||||
`${relativePath}/orphan.vhd`,
|
||||
`${relativePath}/child.vhd`,
|
||||
// abandonned.json is not here
|
||||
// abandoned.json is not here
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
await generateVhd(`${basePath}/abandonned.vhd`)
|
||||
await generateVhd(`${basePath}/abandoned.vhd`)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
@@ -162,12 +162,12 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
|
||||
},
|
||||
})
|
||||
|
||||
let loggued = ''
|
||||
let logged = ''
|
||||
const logInfo = message => {
|
||||
loggued += message + '\n'
|
||||
logged += message + '\n'
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
|
||||
let matched = loggued.match(/deleting unused VHD/g) || []
|
||||
let matched = logged.match(/deleting unused VHD/g) || []
|
||||
assert.equal(matched.length, 1) // only one vhd should have been deleted
|
||||
|
||||
// a missing vhd cause clean to remove all vhds
|
||||
@@ -179,14 +179,14 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
|
||||
`deleted.vhd`, // in metadata but not in vhds
|
||||
`orphan.vhd`,
|
||||
`child.vhd`,
|
||||
// abandonned.vhd is not here anymore
|
||||
// abandoned.vhd is not here anymore
|
||||
],
|
||||
}),
|
||||
{ flags: 'w' }
|
||||
)
|
||||
loggued = ''
|
||||
logged = ''
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: () => {}, lock: false })
|
||||
matched = loggued.match(/deleting unused VHD/g) || []
|
||||
matched = logged.match(/deleting unused VHD/g) || []
|
||||
assert.equal(matched.length, 2) // all vhds (orphan and child ) should have been deleted
|
||||
})
|
||||
|
||||
@@ -212,6 +212,7 @@ test('it merges delta of non destroyed chain', async () => {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUuid: orphan.footer.uuid,
|
||||
},
|
||||
blocks: [0, 1],
|
||||
})
|
||||
// a grand child
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
@@ -221,21 +222,22 @@ test('it merges delta of non destroyed chain', async () => {
|
||||
},
|
||||
})
|
||||
|
||||
let loggued = []
|
||||
let logged = []
|
||||
const logInfo = message => {
|
||||
loggued.push(message)
|
||||
logged.push(message)
|
||||
}
|
||||
await adapter.cleanVm(rootPath, { remove: true, logInfo, logWarn: logInfo, lock: false })
|
||||
assert.equal(loggued[0], `unexpected number of entries in backup cache`)
|
||||
assert.equal(logged[0], `unexpected number of entries in backup cache`)
|
||||
|
||||
loggued = []
|
||||
await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
|
||||
const [merging] = loggued
|
||||
logged = []
|
||||
const result = await adapter.cleanVm(rootPath, { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
|
||||
assert.ok(result.size > 0, `no merged size`)
|
||||
const [merging] = logged
|
||||
assert.equal(merging, `merging VHD chain`)
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children after the merge
|
||||
assert.equal(metadata.size, 104960)
|
||||
assert.equal(metadata.size, 4299776)
|
||||
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
// only check deletion
|
||||
@@ -286,6 +288,54 @@ test('it finish unterminated merge ', async () => {
|
||||
assert.equal(remainingVhds.includes('child.vhd'), true)
|
||||
})
|
||||
|
||||
test('it finishes unterminated merge when child is a disk chain', async () => {
|
||||
// This test covers the case where a merge was interrupted and the child disk is a disk chain.
|
||||
|
||||
await handler.writeFile(
|
||||
`${rootPath}/metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: 209920,
|
||||
vhds: [`${relativePath}/child.vhd`],
|
||||
})
|
||||
)
|
||||
|
||||
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`)
|
||||
const intermediate = await generateVhd(`${basePath}/intermediate.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'ancestor.vhd',
|
||||
parentUuid: ancestor.footer.uuid,
|
||||
},
|
||||
})
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'intermediate.vhd',
|
||||
parentUuid: intermediate.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
// merge state file with chain field
|
||||
await handler.writeFile(
|
||||
`${basePath}/.ancestor.vhd.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: { uuid: '0' },
|
||||
child: { uuid: '0' },
|
||||
chain: ['ancestor.vhd', 'intermediate.vhd', 'child.vhd'],
|
||||
currentBlock: 0,
|
||||
mergedDataSize: 0,
|
||||
step: 'mergeBlocks',
|
||||
diskSize: 0,
|
||||
})
|
||||
)
|
||||
|
||||
await adapter.cleanVm(rootPath, { remove: true, merge: true, logWarn: () => {}, lock: false })
|
||||
|
||||
// ancestor should be renamed to child.vhd (replacing it), intermediate.vhd should be deleted
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
assert.equal(remainingVhds.length, 1)
|
||||
assert.equal(remainingVhds[0], 'child.vhd')
|
||||
})
|
||||
|
||||
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
|
||||
// the message and resulting files should be identical to the output with vhd files which is tested independently
|
||||
describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
@@ -311,7 +361,7 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
// a vhd non referenced in metadata
|
||||
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
|
||||
// an abandoned delta vhd without its parent
|
||||
await generateVhd(`${basePath}/abandonned.vhd`, {
|
||||
await generateVhd(`${basePath}/abandoned.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
@@ -402,13 +452,12 @@ describe('tests multiple combination ', { concurrency: 1 }, () => {
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`${rootPath}/metadata.json`))
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
assert.deepEqual(metadata.size, vhdMode === 'file' ? 6502400 : 6501888)
|
||||
assert.deepEqual(metadata.size, 4404224)
|
||||
|
||||
// broken vhd, non referenced, abandoned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
// grand child and clean vhd should not have changed
|
||||
const survivors = await handler.list(basePath)
|
||||
// console.log(survivors)
|
||||
if (useAlias) {
|
||||
const dataSurvivors = await handler.list(basePath + '/data')
|
||||
// the goal of the alias : do not move a full folder
|
||||
|
||||
@@ -1,40 +1,20 @@
|
||||
import * as UUID from 'uuid'
|
||||
import sum from 'lodash/sum.js'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
|
||||
import { Constants, openVhd, VhdAbstract } from 'vhd-lib'
|
||||
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
import { basename, dirname, resolve } from 'node:path'
|
||||
import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
|
||||
import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { mergeVhdChain } from 'vhd-lib/merge.js'
|
||||
import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
|
||||
import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
|
||||
import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
|
||||
|
||||
import { Task } from './Task.mjs'
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
import { Task } from '@vates/task'
|
||||
import handlerPath from '@xen-orchestra/fs/path'
|
||||
|
||||
const { DISK_TYPES } = Constants
|
||||
|
||||
// checking the size of a vhd directory is costly
|
||||
// 1 Http Query per 1000 blocks
|
||||
// we only check size of all the vhd are VhdFiles
|
||||
function shouldComputeVhdsSize(handler, vhds) {
|
||||
if (handler.isEncrypted) {
|
||||
return false
|
||||
}
|
||||
return vhds.every(vhd => vhd instanceof VhdFile)
|
||||
}
|
||||
|
||||
const computeVhdsSize = (handler, vhdPaths) =>
|
||||
Disposable.use(
|
||||
vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
|
||||
async vhds => {
|
||||
if (shouldComputeVhdsSize(handler, vhds)) {
|
||||
const sizes = await asyncMap(vhds, vhd => vhd.getSize())
|
||||
return sum(sizes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// chain is [ ancestor, child_1, ..., child_n ]
|
||||
async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
|
||||
logInfo(`merging VHD chain`, { chain })
|
||||
@@ -51,7 +31,15 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
|
||||
}
|
||||
}, 10e3)
|
||||
try {
|
||||
return await mergeVhdChain(handler, chain, {
|
||||
const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
|
||||
|
||||
const childDisks = []
|
||||
for (const path of chain) {
|
||||
childDisks.push(new RemoteVhdDisk({ handler, path }))
|
||||
}
|
||||
const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
|
||||
|
||||
const mergeRemoteDisk = new MergeRemoteDisk(handler, {
|
||||
logInfo,
|
||||
mergeBlockConcurrency,
|
||||
onProgress({ done: d, total: t }) {
|
||||
@@ -60,6 +48,14 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
|
||||
},
|
||||
removeUnused: remove,
|
||||
})
|
||||
|
||||
const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
|
||||
await parentDisk.init({ force: isResumingMerge })
|
||||
await childDiskChain.init({ force: isResumingMerge })
|
||||
|
||||
const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
|
||||
|
||||
return result
|
||||
} finally {
|
||||
clearInterval(handle)
|
||||
}
|
||||
@@ -68,10 +64,12 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
|
||||
const noop = Function.prototype
|
||||
|
||||
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
|
||||
|
||||
const listVhds = async (handler, vmDir, logWarn) => {
|
||||
const vhds = new Set()
|
||||
const aliases = {}
|
||||
const interruptedVhds = new Map()
|
||||
const checksums = new Set()
|
||||
|
||||
await asyncMap(
|
||||
await handler.list(`${vmDir}/vdis`, {
|
||||
@@ -85,11 +83,15 @@ const listVhds = async (handler, vmDir, logWarn) => {
|
||||
}),
|
||||
async vdiDir => {
|
||||
const list = await handler.list(vdiDir, {
|
||||
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
|
||||
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
|
||||
})
|
||||
aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
|
||||
|
||||
await asyncMap(list, async file => {
|
||||
if (isVhdSumFile(file)) {
|
||||
checksums.add(`${vdiDir}/${file}`)
|
||||
return
|
||||
}
|
||||
const res = INTERRUPTED_VHDS_REG.exec(file)
|
||||
if (res === null) {
|
||||
vhds.add(`${vdiDir}/${file}`)
|
||||
@@ -111,7 +113,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
|
||||
)
|
||||
)
|
||||
|
||||
return { vhds, interruptedVhds, aliases }
|
||||
return { vhds, interruptedVhds, aliases, checksums }
|
||||
}
|
||||
|
||||
export async function checkAliases(
|
||||
@@ -228,7 +230,16 @@ export async function cleanVm(
|
||||
const vhdParents = { __proto__: null }
|
||||
const vhdChildren = { __proto__: null }
|
||||
|
||||
const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
|
||||
const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
|
||||
|
||||
// from 5.110 to 5.113 we computed checksum for vhd file
|
||||
// but never used nor removed them
|
||||
await asyncMap(checksums, async path => {
|
||||
if (remove) {
|
||||
logInfo('deleting checksum file ', { path })
|
||||
return handler.unlink(path)
|
||||
}
|
||||
})
|
||||
|
||||
// remove broken VHDs
|
||||
await asyncMap(vhds, async path => {
|
||||
@@ -503,23 +514,27 @@ export async function cleanVm(
|
||||
})
|
||||
}
|
||||
|
||||
let totalMergedDataSize = 0
|
||||
const metadataWithMergedVhd = {}
|
||||
const doMerge = async () => {
|
||||
await asyncMap(toMerge, async chain => {
|
||||
const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
|
||||
const { finalDiskSize, mergedDataSize } = await limitedMergeVhdChain(handler, chain, {
|
||||
logInfo,
|
||||
logWarn,
|
||||
remove,
|
||||
mergeBlockConcurrency,
|
||||
})
|
||||
totalMergedDataSize += mergedDataSize
|
||||
const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
|
||||
metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
|
||||
})
|
||||
|
||||
return { size: totalMergedDataSize }
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
...unusedVhdsDeletion,
|
||||
toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : () => Promise.resolve()),
|
||||
toMerge.length !== 0 && (merge ? Task.run({ properties: { name: 'merge' } }, doMerge) : () => Promise.resolve()),
|
||||
asyncMap(unusedXvas, path => {
|
||||
logWarn('unused XVA', { path })
|
||||
if (remove) {
|
||||
@@ -547,7 +562,7 @@ export async function cleanVm(
|
||||
let fileSystemSize
|
||||
const mergedSize = metadataWithMergedVhd[metadataPath]
|
||||
|
||||
const { mode, size, vhds, xva } = metadata
|
||||
const { mode, size, xva } = metadata
|
||||
|
||||
try {
|
||||
if (mode === 'full') {
|
||||
@@ -563,21 +578,7 @@ export async function cleanVm(
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
// can fail with encrypted remote
|
||||
}
|
||||
} else if (mode === 'delta') {
|
||||
// don't warn if the size has changed after a merge
|
||||
if (mergedSize === undefined) {
|
||||
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
|
||||
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
|
||||
// the size is not computed in some cases (e.g. VhdDirectory)
|
||||
if (fileSystemSize !== undefined && fileSystemSize !== size) {
|
||||
logWarn('cleanVm: incorrect backup size in metadata', {
|
||||
path: metadataPath,
|
||||
actual: size ?? 'none',
|
||||
expected: fileSystemSize,
|
||||
})
|
||||
}
|
||||
// will fail with encrypted remote
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -622,5 +623,6 @@ export async function cleanVm(
|
||||
return {
|
||||
// boolean whether some VHDs were merged (or should be merged)
|
||||
merge: toMerge.length !== 0,
|
||||
size: totalMergedDataSize,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,58 @@
|
||||
// @ts-check
|
||||
|
||||
import moment from 'moment-timezone'
|
||||
import assert from 'node:assert'
|
||||
/**
|
||||
* @typedef {import('moment-timezone').Moment} Moment
|
||||
*/
|
||||
/**
|
||||
* @typedef {Object} LTRSettings
|
||||
* @property {number} [firstHourOfTheDay] - First hour of the day (for daily retention)
|
||||
* @property {number} [firstDayOfWeek] - First day of the week (for weekly retention)
|
||||
* @property {number} [firstDayOfMonth] - First day of the month (for monthly retention)
|
||||
* @property {number} [firstDayOfYear] - First day of the year (for yearly retention)
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LTRConfig
|
||||
* @property {number} retention - Number of time buckets to retain
|
||||
* @property {LTRSettings} [settings] - Optional settings for the retention period
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LongTermRetention
|
||||
* @property {LTRConfig} [daily] - Daily retention configuration
|
||||
* @property {LTRConfig} [weekly] - Weekly retention configuration
|
||||
* @property {LTRConfig} [monthly] - Monthly retention configuration
|
||||
* @property {LTRConfig} [yearly] - Yearly retention configuration
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} Entry
|
||||
* @property {number | undefined} timestamp - Unix timestamp of the entry
|
||||
* @property {string} id
|
||||
* @property {*} [key] - Any other properties
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DateBucket
|
||||
* @property {number} remaining - Number of remaining buckets to fill
|
||||
* @property {string|null} lastMatchingBucket - Last bucket key that was matched
|
||||
* @property {(date: Moment) => string} formatter - Function to format date into bucket key
|
||||
* @property {Object.<string, Entry>} entries - Map of bucket keys to entries
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} LTRDefinition
|
||||
* @property {(options: {firstHourOfTheDay?: number, firstDayOfWeek?: number, firstDayOfMonth?: number, firstDayOfYear?: number, dateCreator: (date: Moment) => Moment}) => (date: Moment) => string} makeDateFormatter
|
||||
* @property {string} [ancestor] - Name of the parent retention period
|
||||
*/
|
||||
|
||||
/**
|
||||
* Creates a function that converts dates to moment objects in a specific timezone
|
||||
* @param {string|undefined} timezone - IANA timezone string (e.g., 'America/New_York')
|
||||
* @returns {(date: Moment | number) => Moment} Function that creates timezoned moment objects
|
||||
*/
|
||||
|
||||
function instantiateTimezonedDateCreator(timezone) {
|
||||
return date => {
|
||||
@@ -9,9 +62,12 @@ function instantiateTimezonedDateCreator(timezone) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {Object.<string, LTRDefinition>}
|
||||
*/
|
||||
const LTR_DEFINITIONS = {
|
||||
daily: {
|
||||
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.hour(copy.hour() - firstHourOfTheDay)
|
||||
@@ -20,11 +76,11 @@ const LTR_DEFINITIONS = {
|
||||
},
|
||||
},
|
||||
weekly: {
|
||||
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
|
||||
copy.date(date.date() - firstDayOfWeek)
|
||||
copy.date(copy.date() - firstDayOfWeek)
|
||||
// warning, the year in term of week may different from YYYY
|
||||
// since the computation of the first week of a year is timezone dependent
|
||||
return copy.format('gggg-ww')
|
||||
@@ -33,7 +89,7 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'daily',
|
||||
},
|
||||
monthly: {
|
||||
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.date(copy.date() - firstDayOfMonth)
|
||||
@@ -43,7 +99,7 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'weekly',
|
||||
},
|
||||
yearly: {
|
||||
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
|
||||
makeDateFormatter: ({ firstDayOfYear = 0, dateCreator }) => {
|
||||
return date => {
|
||||
const copy = dateCreator(date)
|
||||
copy.date(copy.date() - firstDayOfYear)
|
||||
@@ -53,27 +109,23 @@ const LTR_DEFINITIONS = {
|
||||
ancestor: 'monthly',
|
||||
},
|
||||
}
|
||||
|
||||
// returns all entries but the last retention-th
|
||||
/**
|
||||
* return the entries too old to be kept
|
||||
* if multiple entries are i the same time bucket : keep only the most recent one
|
||||
* if an entry is valid in any of the bucket OR the minRetentionCount : keep it
|
||||
* if a bucket is completely empty : it does not count as one, thus it may extend the retention
|
||||
* @returns Array<Backup>
|
||||
* Groups entries into date buckets based on long-term retention configuration
|
||||
* @template {Entry} T
|
||||
* @param {Array<T>} entries - Array of entries sorted in descending order by timestamp
|
||||
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
|
||||
* @param {string | undefined} timezone - IANA timezone string
|
||||
* @returns {Object.<string, DateBucket>} Map of retention period names to their date buckets
|
||||
*/
|
||||
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
|
||||
assert.strictEqual(
|
||||
typeof minRetentionCount,
|
||||
'number',
|
||||
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
assert.strictEqual(
|
||||
minRetentionCount >= 0,
|
||||
true,
|
||||
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
|
||||
export function getLtrEntries(entries, longTermRetention, timezone) {
|
||||
/**
|
||||
* @type {Object.<string, DateBucket>}
|
||||
*/
|
||||
const dateBuckets = {}
|
||||
if (!longTermRetention || Object.keys(longTermRetention).length === 0) {
|
||||
return {}
|
||||
}
|
||||
const dateCreator = instantiateTimezonedDateCreator(timezone)
|
||||
// only check buckets that have a retention set
|
||||
for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
|
||||
@@ -81,7 +133,7 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
|
||||
assert.notStrictEqual(
|
||||
timezone,
|
||||
undefined,
|
||||
`timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
|
||||
`timezone must defined for ltr, got ${JSON.stringify({ longTermRetention, timezone })}`
|
||||
)
|
||||
assert.strictEqual(
|
||||
typeof retention,
|
||||
@@ -97,40 +149,106 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
|
||||
remaining: retention,
|
||||
lastMatchingBucket: null,
|
||||
formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
|
||||
entries: {}, // bucket => entry id
|
||||
}
|
||||
}
|
||||
const nb = entries.length
|
||||
const oldEntries = []
|
||||
|
||||
let previousTimestamp = -1
|
||||
for (let i = nb - 1; i >= 0; i--) {
|
||||
const entry = entries[i]
|
||||
// force it so the compiler is sure we have a timestamp
|
||||
if (entry.timestamp === undefined) {
|
||||
assert.notStrictEqual(
|
||||
entry.timestamp,
|
||||
undefined,
|
||||
`Can't compute long term retention if entries don't have a timestamp`
|
||||
)
|
||||
continue
|
||||
}
|
||||
// we go through the entries from the last (most recent) to the first (oldest)
|
||||
assert.ok(
|
||||
previousTimestamp === -1 || entry.timestamp < previousTimestamp,
|
||||
`entries must be sorted in desc order ${new Date(entry.timestamp)} , previous : > ${new Date(previousTimestamp)} `
|
||||
)
|
||||
previousTimestamp = entry.timestamp
|
||||
const entryDate = dateCreator(entry.timestamp)
|
||||
let shouldBeKept = false
|
||||
for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
|
||||
if (remaining === 0) {
|
||||
continue
|
||||
}
|
||||
const bucket = formatter(entryDate)
|
||||
if (lastMatchingBucket !== bucket) {
|
||||
if (lastMatchingBucket !== null) {
|
||||
assert.strictEqual(
|
||||
lastMatchingBucket > bucket,
|
||||
true,
|
||||
`entries must be sorted in asc order ${lastMatchingBucket} ${bucket}`
|
||||
)
|
||||
if (remaining === 0) {
|
||||
continue
|
||||
}
|
||||
shouldBeKept = true
|
||||
dateBuckets[duration].remaining -= 1
|
||||
dateBuckets[duration].lastMatchingBucket = bucket
|
||||
dateBuckets[duration].remaining -= 1
|
||||
}
|
||||
}
|
||||
if (i >= nb - minRetentionCount) {
|
||||
shouldBeKept = true
|
||||
}
|
||||
if (!shouldBeKept) {
|
||||
oldEntries.push(entry)
|
||||
dateBuckets[duration].entries[bucket] = entry
|
||||
}
|
||||
}
|
||||
// we expect the entries to be in the right order
|
||||
return oldEntries.reverse()
|
||||
|
||||
return dateBuckets
|
||||
}
|
||||
|
||||
/**
|
||||
* return the status of an entry
|
||||
* @param {Array<Entry>} entries
|
||||
* @param {Entry} entry
|
||||
* @param {LongTermRetention} longTermRetention - Configuration for retention periods
|
||||
* @param {string | undefined} timezone - IANA timezone string
|
||||
* @returns {Array<{duration: string, dateBucket:string}>}
|
||||
*/
|
||||
export function getEntryStatus(entries, entry, longTermRetention, timezone) {
|
||||
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
|
||||
const entryBuckets = []
|
||||
for (const [duration, { entries }] of Object.entries(dateBuckets)) {
|
||||
for (const [dateBucket, bucketEntry] of Object.entries(entries)) {
|
||||
if (entry.id === bucketEntry.id) {
|
||||
entryBuckets.push({ duration, dateBucket })
|
||||
}
|
||||
}
|
||||
}
|
||||
return entryBuckets
|
||||
}
|
||||
|
||||
/**
|
||||
* return the entries too old to be kept
|
||||
* if multiple entries are in the same time bucket: keep only the oldest one
|
||||
* if an entry is valid in any of the bucket OR the minRetentionCount: keep it
|
||||
* if a bucket is completely empty: it does not count as one, thus it may extend the retention
|
||||
* if an entry is within the most recents minRetentionCount entries : keep it
|
||||
*
|
||||
* @param {number} minRetentionCount - Minimum number of most recent entries to always keep
|
||||
* @param {Array<Entry>} entries - Array of entries sorted in descending order by timestamp
|
||||
* @param {Object} options - Additional options
|
||||
* @param {LongTermRetention} [options.longTermRetention={}] - Configuration for retention periods
|
||||
* @param {string | undefined} [options.timezone] - IANA timezone string
|
||||
* @returns {Array<Entry>} Array of entries that can be removed
|
||||
*/
|
||||
export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
|
||||
assert.strictEqual(
|
||||
typeof minRetentionCount,
|
||||
'number',
|
||||
`minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
assert.ok(
|
||||
minRetentionCount >= 0,
|
||||
`minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
|
||||
)
|
||||
const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
|
||||
|
||||
const kept = new Set(entries.filter((_, index) => index >= entries.length - minRetentionCount))
|
||||
/**
|
||||
* @type {Set<Entry>}
|
||||
*/
|
||||
for (const { entries } of Object.values(dateBuckets)) {
|
||||
for (const entry of Object.values(entries)) {
|
||||
kept.add(entry)
|
||||
}
|
||||
}
|
||||
|
||||
// ensure order is the same as the source
|
||||
const oldEntries = entries.filter((entry, index) => {
|
||||
return !kept.has(entry)
|
||||
})
|
||||
|
||||
return oldEntries
|
||||
}
|
||||
|
||||
@@ -10,24 +10,28 @@ describe('_getOldEntries() should succeed', () => {
|
||||
1,
|
||||
[
|
||||
{ timestamp: 1, id: 1 },
|
||||
{ timestamp: 3, id: 2 },
|
||||
{ timestamp: 2, id: 3 },
|
||||
{ timestamp: 2, id: 2 },
|
||||
{ timestamp: 3, id: 3 },
|
||||
],
|
||||
],
|
||||
expectedIds: [1, 2],
|
||||
testLabel: 'should handle number based retention ',
|
||||
},
|
||||
|
||||
{
|
||||
args: [3, [{ id: 1 }, 4, new Date(), 'whatever']],
|
||||
expectedIds: [1],
|
||||
testLabel: 'should handle number based retention without timestamp ',
|
||||
},
|
||||
{
|
||||
args: [
|
||||
0,
|
||||
[
|
||||
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // too old
|
||||
{ timestamp: +new Date('2024-09-01 00:00:00'), id: 2 }, // too old
|
||||
{ timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // too old in same day
|
||||
{ timestamp: +new Date('2024-09-01 00:00:00'), id: 1 }, // too old
|
||||
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 2 }, // too old
|
||||
{ timestamp: +new Date('2024-09-02 00:09:00'), id: 3 }, // oldest in same day
|
||||
{ timestamp: +new Date('2024-09-02 00:10:00'), id: 4 },
|
||||
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 5 },
|
||||
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // too old in same day
|
||||
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // oldest in same day
|
||||
{ timestamp: +new Date('2024-09-04 00:10:00'), id: 7 },
|
||||
],
|
||||
{
|
||||
@@ -37,22 +41,22 @@ describe('_getOldEntries() should succeed', () => {
|
||||
timezone: 'Europe/Paris',
|
||||
},
|
||||
],
|
||||
expectedIds: [1, 2, 3, 6],
|
||||
expectedIds: [1, 2, 4, 7],
|
||||
testLabel: 'should handle day based retention ',
|
||||
},
|
||||
{
|
||||
args: [
|
||||
0,
|
||||
[
|
||||
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // week n-3 too old
|
||||
{ timestamp: +new Date('2024-09-02 00:00:00'), id: 2 }, // week n-3 too old
|
||||
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2
|
||||
{ timestamp: +new Date('2024-08-30 00:01:00'), id: 1 }, // week n-3 too old
|
||||
{ timestamp: +new Date('2024-08-31 00:00:00'), id: 2 }, // week n-3 too old
|
||||
{ timestamp: +new Date('2024-09-03 00:09:00'), id: 3 }, // week n-2, oldest
|
||||
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 4 }, // week n-2
|
||||
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // week n-2
|
||||
{ timestamp: +new Date('2024-09-06 00:09:00'), id: 6 }, // week n-2
|
||||
{ timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2, most recent kept
|
||||
{ timestamp: +new Date('2024-09-09 00:09:00'), id: 8 }, // week n-1, too old
|
||||
{ timestamp: +new Date('2024-09-15 00:09:00'), id: 9 }, // week n-1 kept
|
||||
{ timestamp: +new Date('2024-09-07 00:09:00'), id: 7 }, // week n-2
|
||||
{ timestamp: +new Date('2024-09-15 00:09:00'), id: 8 }, // week n-1, oldest
|
||||
{ timestamp: +new Date('2024-09-21 00:09:00'), id: 9 }, // week n-1 most recent
|
||||
{ timestamp: +new Date('2024-09-22 00:09:00'), id: 10 }, // week n kept
|
||||
],
|
||||
{
|
||||
@@ -62,7 +66,7 @@ describe('_getOldEntries() should succeed', () => {
|
||||
timezone: 'Europe/Paris',
|
||||
},
|
||||
],
|
||||
expectedIds: [1, 2, 3, 4, 5, 6, 8],
|
||||
expectedIds: [1, 2, 4, 5, 6, 7, 9],
|
||||
testLabel: 'should handle week based retention ',
|
||||
},
|
||||
{
|
||||
@@ -85,26 +89,26 @@ describe('_getOldEntries() should succeed', () => {
|
||||
timezone: 'Europe/Paris',
|
||||
},
|
||||
],
|
||||
expectedIds: [1, 3, 4, 6, 7],
|
||||
expectedIds: [1, 4, 5, 7, 8],
|
||||
testLabel: 'should handle month based retention',
|
||||
},
|
||||
{
|
||||
args: [
|
||||
0,
|
||||
2,
|
||||
[
|
||||
{ timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, // too old
|
||||
{ timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // too old in same year
|
||||
{ timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, // too old
|
||||
{ timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, // too old
|
||||
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, // too old
|
||||
{ timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, // new month,
|
||||
{ timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, // new month , week reached retention
|
||||
{ timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, // new week
|
||||
{ timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, // new week/month / year daily reach retention
|
||||
{ timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, // same day/week/month/year
|
||||
{ timestamp: +new Date('2025-01-01 00:10:00'), id: 11 }, // new day / week / month
|
||||
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // same day/week/month/year
|
||||
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 13 }, // new month /year
|
||||
{ timestamp: +new Date('2023-05-18 00:09:00'), id: 1 }, //
|
||||
{ timestamp: +new Date('2024-06-15 00:09:00'), id: 2 }, // year n -1
|
||||
{ timestamp: +new Date('2024-07-04 00:09:00'), id: 3 }, //
|
||||
{ timestamp: +new Date('2024-08-12 00:09:00'), id: 4 }, //
|
||||
{ timestamp: +new Date('2024-09-05 00:09:00'), id: 5 }, //
|
||||
{ timestamp: +new Date('2024-10-02 00:09:00'), id: 6 }, // month n-4
|
||||
{ timestamp: +new Date('2024-11-01 00:09:00'), id: 7 }, // month n-3
|
||||
{ timestamp: +new Date('2024-12-17 00:09:00'), id: 8 }, // week n-3
|
||||
{ timestamp: +new Date('2024-12-24 00:09:00'), id: 9 }, // week n-2, month n-2,
|
||||
{ timestamp: +new Date('2025-01-01 00:09:00'), id: 10 }, // week n-1, year n
|
||||
{ timestamp: +new Date('2025-01-02 00:10:00'), id: 11 }, // day n-1, month n-1,
|
||||
{ timestamp: +new Date('2025-12-31 00:09:00'), id: 12 }, // day n , week n , month n
|
||||
{ timestamp: +new Date('2025-12-31 10:09:00'), id: 13 }, // min retention
|
||||
],
|
||||
{
|
||||
longTermRetention: {
|
||||
@@ -116,29 +120,29 @@ describe('_getOldEntries() should succeed', () => {
|
||||
timezone: 'Europe/Paris', // use a time zone here because week definition is timezone bound
|
||||
},
|
||||
],
|
||||
expectedIds: [1, 2, 3, 4, 5, 10, 12],
|
||||
expectedIds: [1, 3, 4, 5],
|
||||
testLabel: 'complete test ',
|
||||
},
|
||||
{
|
||||
args: [
|
||||
0,
|
||||
[
|
||||
{ timestamp: +new Date('2024-09-01 00:01:00'), id: 1 }, // thrid day too old
|
||||
{ timestamp: +new Date('2024-09-01 00:10:00'), id: 2 }, // thrid day
|
||||
{ timestamp: +new Date('2024-09-01 00:20:00'), id: 3 }, // second day, too old
|
||||
{ timestamp: +new Date('2024-09-02 00:22:00'), id: 4 }, // second day too old
|
||||
{ timestamp: +new Date('2024-09-03 00:20:00'), id: 5 }, // second day in NZ
|
||||
{ timestamp: +new Date('2024-09-04 00:09:00'), id: 6 }, // same day in NZ
|
||||
{ timestamp: +new Date('2024-09-04 00:10:00'), id: 7 }, // most recent
|
||||
{ timestamp: +new Date('2024-08-31T09:01:00Z'), id: 1 }, // 4th day too old
|
||||
{ timestamp: +new Date('2024-09-01T00:10:00Z'), id: 2 }, // third day
|
||||
{ timestamp: +new Date('2024-09-01T14:20:00Z'), id: 3 }, // second day, oldest
|
||||
{ timestamp: +new Date('2024-09-02T00:22:00Z'), id: 4 }, // second day
|
||||
{ timestamp: +new Date('2024-09-03T14:00:00Z'), id: 5 }, // same day oldest
|
||||
{ timestamp: +new Date('2024-09-04T00:09:00Z'), id: 6 }, // same day in NZ
|
||||
{ timestamp: +new Date('2024-09-04T00:10:00Z'), id: 7 }, // most recent
|
||||
],
|
||||
{
|
||||
longTermRetention: {
|
||||
daily: { retention: 3 },
|
||||
},
|
||||
timezone: 'Pacific/Auckland', // GMT +6
|
||||
timezone: 'Pacific/Auckland', // GMT +12 (NZST in September)
|
||||
},
|
||||
],
|
||||
expectedIds: [1, 2, 3, 6],
|
||||
expectedIds: [1, 4, 6, 7],
|
||||
testLabel: 'should handle timezone ',
|
||||
},
|
||||
]
|
||||
@@ -149,13 +153,20 @@ describe('_getOldEntries() should succeed', () => {
|
||||
assert.strictEqual(
|
||||
oldEntries.length,
|
||||
expectedIds.length,
|
||||
`different length , ${JSON.stringify({ oldEntries, expectedIds })}`
|
||||
`different length , ${JSON.stringify({ expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
|
||||
)
|
||||
for (let i = 0; i < expectedIds.length; i++) {
|
||||
if (i > 0) {
|
||||
assert.strictEqual(
|
||||
expectedIds[i - 1] < expectedIds[i],
|
||||
true,
|
||||
`expect ids in ascending order ${expectedIds[i - 1]} ${expectedIds[i]}`
|
||||
)
|
||||
}
|
||||
assert.strictEqual(
|
||||
oldEntries[i].id,
|
||||
expectedIds[i],
|
||||
`different id , ${JSON.stringify({ i, expectedIds, oldEntries })}`
|
||||
`different id , ${JSON.stringify({ i, expectedIds, oldEntries: oldEntries.map(({ id }) => id) })}`
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -272,6 +283,19 @@ describe('_getOldEntries() should fail when called incorrectly', () => {
|
||||
],
|
||||
testLabel: 'broken date ',
|
||||
},
|
||||
{
|
||||
args: [
|
||||
1,
|
||||
[{ obj: 1 }, 'toto', Date.now()],
|
||||
{
|
||||
longTermRetention: {
|
||||
daily: { retention: 5 },
|
||||
},
|
||||
timezone: 'Europe/Paris',
|
||||
},
|
||||
],
|
||||
testLabel: 'should fail if LTR without timestamp',
|
||||
},
|
||||
]
|
||||
|
||||
for (const { args, testLabel } of tests) {
|
||||
|
||||
@@ -4,16 +4,15 @@ import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { CancelToken } from 'promise-toolbox'
|
||||
import { compareVersions } from 'compare-versions'
|
||||
import { defer } from 'golike-defer'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { cancelableMap } from './_cancelableMap.mjs'
|
||||
import { Task } from './Task.mjs'
|
||||
import pick from 'lodash/pick.js'
|
||||
import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
|
||||
import { BASE_DELTA_VDI, CONTENT_KEY, COPY_OF, VM_UUID } from './_otherConfig.mjs'
|
||||
|
||||
import { XapiDiskSource } from '@xen-orchestra/xapi'
|
||||
import { VHD_MAX_SIZE, XapiDiskSource } from '@xen-orchestra/xapi'
|
||||
import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
|
||||
import { toQcow2Stream } from '@xen-orchestra/qcow2'
|
||||
import { VHD_MAX_SIZE } from '@xen-orchestra/xapi/disks/Xapi.mjs'
|
||||
|
||||
const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
|
||||
|
||||
@@ -125,7 +124,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
$defer,
|
||||
incrementalVm,
|
||||
sr,
|
||||
{ cancelToken = CancelToken.none, newMacAddresses = false } = {}
|
||||
{ cancelToken = CancelToken.none, newMacAddresses = false, targetRef = undefined } = {}
|
||||
) {
|
||||
const { version } = incrementalVm
|
||||
if (compareVersions(version, '1.0.0') < 0) {
|
||||
@@ -137,44 +136,106 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
|
||||
const vdiRecords = incrementalVm.vdis
|
||||
|
||||
// 0. Create suspend_VDI
|
||||
// When targetRef is provided, update the existing VM instead of creating a new one.
|
||||
const targetVm = targetRef !== undefined ? xapi.getObjectByRef(targetRef, undefined) : undefined
|
||||
const isUpdate = targetVm !== undefined && !targetVm.is_a_snapshot && !targetVm.is_a_template
|
||||
|
||||
// 0. Create suspend_VDI (only when creating a new VM).
|
||||
let suspendVdi
|
||||
if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
|
||||
const vdi = vdiRecords[vmRecord.suspend_VDI]
|
||||
if (vdi === undefined) {
|
||||
Task.warning('Suspend VDI not available for this suspended VM', {
|
||||
vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
|
||||
})
|
||||
} else {
|
||||
suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
|
||||
$defer.onFailure(() => suspendVdi.$destroy())
|
||||
if (!isUpdate) {
|
||||
if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
|
||||
const vdi = vdiRecords[vmRecord.suspend_VDI]
|
||||
if (vdi === undefined) {
|
||||
Task.warning('Suspend VDI not available for this suspended VM', {
|
||||
vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
|
||||
})
|
||||
} else {
|
||||
suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
|
||||
$defer.onFailure(() => suspendVdi.$destroy())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Create the VM.
|
||||
const vmRef = await xapi.VM_create(
|
||||
{
|
||||
...vmRecord,
|
||||
affinity: undefined,
|
||||
blocked_operations: {
|
||||
...vmRecord.blocked_operations,
|
||||
// 1. Create the VM or update the existing one.
|
||||
let vmRef
|
||||
if (isUpdate) {
|
||||
vmRef = targetRef
|
||||
await Promise.all([
|
||||
xapi.setFields('VM', vmRef, {
|
||||
actions_after_crash: vmRecord.actions_after_crash,
|
||||
actions_after_reboot: vmRecord.actions_after_reboot,
|
||||
actions_after_shutdown: vmRecord.actions_after_shutdown,
|
||||
domain_type: vmRecord.domain_type,
|
||||
ha_restart_priority: vmRecord.ha_restart_priority,
|
||||
has_vendor_device: vmRecord.has_vendor_device,
|
||||
HVM_boot_params: vmRecord.HVM_boot_params,
|
||||
HVM_boot_policy: vmRecord.HVM_boot_policy,
|
||||
HVM_shadow_multiplier: vmRecord.HVM_shadow_multiplier,
|
||||
memory_dynamic_max: vmRecord.memory_dynamic_max,
|
||||
memory_dynamic_min: vmRecord.memory_dynamic_min,
|
||||
memory_static_max: vmRecord.memory_static_max,
|
||||
memory_static_min: vmRecord.memory_static_min,
|
||||
name_label: vmRecord.name_label,
|
||||
name_description: vmRecord.name_description,
|
||||
order: vmRecord.order,
|
||||
other_config: vmRecord.other_config,
|
||||
platform: vmRecord.platform,
|
||||
PV_args: vmRecord.PV_args,
|
||||
PV_bootloader: vmRecord.PV_bootloader,
|
||||
PV_bootloader_args: vmRecord.PV_bootloader_args,
|
||||
PV_kernel: vmRecord.PV_kernel,
|
||||
PV_legacy_args: vmRecord.PV_legacy_args,
|
||||
PV_ramdisk: vmRecord.PV_ramdisk,
|
||||
recommendations: vmRecord.recommendations,
|
||||
shutdown_delay: vmRecord.shutdown_delay,
|
||||
start_delay: vmRecord.start_delay,
|
||||
suspend_SR: vmRecord.suspend_SR,
|
||||
tags: vmRecord.tags,
|
||||
user_version: vmRecord.user_version,
|
||||
VCPUs_at_startup: vmRecord.VCPUs_at_startup,
|
||||
VCPUs_max: vmRecord.VCPUs_max,
|
||||
VCPUs_params: vmRecord.VCPUs_params,
|
||||
xenstore_data: vmRecord.xenstore_data,
|
||||
}),
|
||||
targetVm.update_blocked_operations({
|
||||
start: 'Importing…',
|
||||
start_on: 'Importing…',
|
||||
}),
|
||||
])
|
||||
} else {
|
||||
vmRef = await xapi.VM_create(
|
||||
{
|
||||
...vmRecord,
|
||||
affinity: undefined,
|
||||
blocked_operations: {
|
||||
...vmRecord.blocked_operations,
|
||||
start: 'Importing…',
|
||||
start_on: 'Importing…',
|
||||
},
|
||||
ha_always_run: false,
|
||||
is_a_template: false,
|
||||
name_label: '[Importing…] ' + vmRecord.name_label,
|
||||
},
|
||||
ha_always_run: false,
|
||||
is_a_template: false,
|
||||
name_label: '[Importing…] ' + vmRecord.name_label,
|
||||
},
|
||||
{
|
||||
bios_strings: vmRecord.bios_strings,
|
||||
generateMacSeed: newMacAddresses,
|
||||
suspend_VDI: suspendVdi?.$ref,
|
||||
}
|
||||
)
|
||||
$defer.onFailure.call(xapi, 'VM_destroy', vmRef)
|
||||
{
|
||||
bios_strings: vmRecord.bios_strings,
|
||||
generateMacSeed: newMacAddresses,
|
||||
suspend_VDI: suspendVdi?.$ref,
|
||||
}
|
||||
)
|
||||
$defer.onFailure.call(xapi, 'VM_destroy', vmRef)
|
||||
}
|
||||
|
||||
// 2. Delete all VBDs which may have been created by the import.
|
||||
await asyncMap(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
|
||||
// In update mode, also collect the VDI refs so orphaned VDIs can be destroyed after new ones are created.
|
||||
const existingVbdRefs = await xapi.getField('VM', vmRef, 'VBDs')
|
||||
const oldVdiRefs = new Set(
|
||||
isUpdate
|
||||
? (await asyncMap(existingVbdRefs, ref => xapi.getField('VBD', ref, 'VDI'))).filter(
|
||||
ref => ref !== undefined && ref !== 'OpaqueRef:NULL'
|
||||
)
|
||||
: []
|
||||
)
|
||||
await asyncMap(existingVbdRefs, ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
|
||||
|
||||
// 3. Create VDIs & VBDs.
|
||||
const vbdRecords = incrementalVm.vbds
|
||||
@@ -185,10 +246,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
let newVdi
|
||||
|
||||
if (vdi.baseVdi?.$ref !== undefined) {
|
||||
newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
|
||||
$defer.onFailure(() => newVdi.$destroy())
|
||||
|
||||
if (isUpdate) {
|
||||
// In update mode, reuse the existing target VDI directly — no clone needed.
|
||||
newVdi = vdi.baseVdi
|
||||
oldVdiRefs.delete(newVdi.$ref)
|
||||
} else {
|
||||
newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
|
||||
$defer.onFailure(() => newVdi.$destroy())
|
||||
}
|
||||
await newVdi.update_other_config(COPY_OF, vdi.uuid)
|
||||
if (vdi.other_config[CONTENT_KEY] !== undefined) {
|
||||
await newVdi.update_other_config(CONTENT_KEY, vdi.other_config[CONTENT_KEY])
|
||||
}
|
||||
if (vdi.virtual_size > newVdi.virtual_size) {
|
||||
await newVdi.$callAsync('resize', vdi.virtual_size)
|
||||
}
|
||||
@@ -214,6 +283,16 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
newVdis[vdiRef] = newVdi
|
||||
})
|
||||
|
||||
// 3.5. Destroy old VDIs that are no longer attached to the VM.
|
||||
// Uses ignoreErrors because some storage backends refuse to destroy a VDI that still has snapshot children;
|
||||
// those VDIs will become truly orphaned once the old snapshot VMs are cleaned up by _deleteOldEntries.
|
||||
await asyncMap([...oldVdiRefs], ref => ignoreErrors.call(xapi.call('VDI.destroy', ref)))
|
||||
|
||||
// 4. For updates, destroy existing VIFs before recreating them.
|
||||
if (isUpdate) {
|
||||
await asyncMap(await xapi.getField('VM', vmRef, 'VIFs'), ref => ignoreErrors.call(xapi.call('VIF.destroy', ref)))
|
||||
}
|
||||
|
||||
const networksByNameLabelByVlan = {}
|
||||
let defaultNetwork
|
||||
Object.values(xapi.objects.all).forEach(object => {
|
||||
@@ -286,16 +365,26 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
|
||||
}
|
||||
}),
|
||||
])
|
||||
// recreate VTPMs
|
||||
await Promise.all(
|
||||
(incrementalVm.vtpms ?? []).map(async contents => {
|
||||
await xapi.VTPM_create({ VM: vmRef, contents })
|
||||
})
|
||||
)
|
||||
|
||||
// recreate vtpm (note there is normally only one VTPM per VM at most)
|
||||
const existingVtpmRefs = await xapi.getField('VM', vmRef, 'VTPMs')
|
||||
for (const vtpmRef of existingVtpmRefs ?? []) {
|
||||
await xapi.call('VTPM_destroy', vtpmRef)
|
||||
}
|
||||
for (const contents of incrementalVm.vtpms ?? []) {
|
||||
await xapi.VTPM_create({ VM: vmRef, contents })
|
||||
}
|
||||
|
||||
const vm = await xapi.getRecord('VM', vmRef)
|
||||
await Promise.all([
|
||||
incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
|
||||
xapi.setField('VM', vmRef, 'name_label', incrementalVm.vm.name_label),
|
||||
vmRecord.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
|
||||
xapi.setField('VM', vmRef, 'name_label', vmRecord.name_label),
|
||||
// correctly unlock the VM and reapply the target blocked operations
|
||||
vm.update_blocked_operations({
|
||||
start: null,
|
||||
start_on: null,
|
||||
...vmRecord.blocked_operations,
|
||||
}),
|
||||
])
|
||||
|
||||
return vmRef
|
||||
|
||||
@@ -20,13 +20,31 @@ const IGNORED_PARTITION_TYPES = {
|
||||
0xd5: true,
|
||||
|
||||
0x82: true, // swap
|
||||
// GPT Linux swap GUID
|
||||
'0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': true,
|
||||
}
|
||||
|
||||
export const LVM_PARTITION_TYPE = 0x8e
|
||||
// MBR LVM type
|
||||
export const LVM_PARTITION_TYPE_MBR = 0x8e
|
||||
// GPT LVM type
|
||||
export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
|
||||
|
||||
const parsePartxLine = createParser({
|
||||
keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
|
||||
valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
|
||||
valueTransform: (value, key) => {
|
||||
if (key === 'start' || key === 'size') {
|
||||
return +value
|
||||
}
|
||||
// For GPT partitions, type is a UUID string
|
||||
if (key === 'type' && !value.startsWith('0x')) {
|
||||
return value.toLowerCase()
|
||||
}
|
||||
// For MBR partitions, type is a hex number as string (e.g., "0x8e")
|
||||
if (key === 'type' && value.startsWith('0x')) {
|
||||
return parseInt(value, 16)
|
||||
}
|
||||
return value
|
||||
},
|
||||
})
|
||||
|
||||
// returns an empty array in case of a non-partitioned disk
|
||||
|
||||
@@ -28,6 +28,11 @@ export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
|
||||
// the VM ( not the snapshot) uuid
|
||||
export const VM_UUID = 'xo:backup:vm'
|
||||
|
||||
// in `other_config` of source snapshot VDIs and replicated target VDIs
|
||||
// contains the UUID of the source snapshot VDI whose content this VDI represents
|
||||
// allows direction-agnostic discovery of common base VDIs (e.g. for failback/reverse replication)
|
||||
export const CONTENT_KEY = 'xo:backup:contentKey'
|
||||
|
||||
async function listVdiRefs(xapi, vmRef) {
|
||||
return xapi.VM_getDisks(vmRef)
|
||||
}
|
||||
@@ -70,7 +75,8 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
|
||||
|
||||
/**
|
||||
*
|
||||
* Reset the other_config field of a VM and its VDIs
|
||||
* Reset the other_config field related to backups of a VM and its VDIs
|
||||
*
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
@@ -79,6 +85,7 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
|
||||
export function resetVmOtherConfig(xapi, vmRef) {
|
||||
return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
|
||||
return xapi.setFieldEntries(type, ref, 'other_config', {
|
||||
[COPY_OF]: null,
|
||||
[DATETIME]: null,
|
||||
[DELTA_CHAIN_LENGTH]: null,
|
||||
[EXPORTED_SUCCESSFULLY]: null,
|
||||
@@ -90,39 +97,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* used to ensure compatibility with the previous snapshots that were having the config stored only into VM
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} vmRef
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export async function populateVdisOtherConfig(xapi, vmRef) {
|
||||
const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
|
||||
const {
|
||||
[DATETIME]: datetime,
|
||||
[DELTA_CHAIN_LENGTH]: chainLength,
|
||||
[EXPORTED_SUCCESSFULLY]: successfully,
|
||||
[JOB_ID]: jobId,
|
||||
[REPLICATED_TO_SR_UUID]: replicatedTo,
|
||||
[SCHEDULE_ID]: scheduleId,
|
||||
[VM_UUID]: vmUuid,
|
||||
} = otherConfig
|
||||
|
||||
return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
|
||||
xapi.setFieldEntries(type, ref, 'other_config', {
|
||||
[DATETIME]: datetime,
|
||||
[DELTA_CHAIN_LENGTH]: chainLength,
|
||||
[EXPORTED_SUCCESSFULLY]: successfully,
|
||||
[JOB_ID]: jobId,
|
||||
[REPLICATED_TO_SR_UUID]: replicatedTo,
|
||||
[SCHEDULE_ID]: scheduleId,
|
||||
[VM_UUID]: vmUuid,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* set the other_config key related to a backup of a VM and its associated VDIs
|
||||
@@ -163,3 +137,24 @@ export async function markExportSuccessfull(xapi, vmRef) {
|
||||
xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Set CONTENT_KEY on each VDI of a VM snapshot to its own UUID.
|
||||
*
|
||||
* This marks each snapshot VDI with a unique content identifier that can be
|
||||
* propagated to replicated copies for direction-agnostic base VDI discovery
|
||||
* (e.g. for failback / reverse replication).
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {String} snapshotRef
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export async function setVmSnapshotContentKeys(xapi, snapshotRef) {
|
||||
return applyToVmAndVdis(xapi, snapshotRef, async (type, ref) => {
|
||||
if (type !== 'VDI') {
|
||||
return
|
||||
}
|
||||
const uuid = await xapi.getField(type, ref, 'uuid')
|
||||
return xapi.setFieldEntry(type, ref, 'other_config', CONTENT_KEY, uuid)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { Task } from '@vates/task'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
|
||||
@@ -6,9 +7,10 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
|
||||
import { PoolMetadataBackup } from './_PoolMetadataBackup.mjs'
|
||||
import { XoMetadataBackup } from './_XoMetadataBackup.mjs'
|
||||
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
|
||||
import { runTask } from './_runTask.mjs'
|
||||
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
const DEFAULT_METADATA_SETTINGS = {
|
||||
retentionPoolMetadata: 0,
|
||||
retentionXoMetadata: 0,
|
||||
@@ -55,13 +57,16 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
||||
poolIds.map(id =>
|
||||
this._getRecord('pool', id).catch(error => {
|
||||
// See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
|
||||
runTask(
|
||||
Task.run(
|
||||
{
|
||||
name: 'get pool record',
|
||||
data: { type: 'pool', id },
|
||||
properties: {
|
||||
id,
|
||||
name: 'get pool record',
|
||||
type: 'pool',
|
||||
},
|
||||
},
|
||||
() => Promise.reject(error)
|
||||
)
|
||||
).catch(noop)
|
||||
})
|
||||
)
|
||||
),
|
||||
@@ -81,11 +86,11 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
||||
if (pools.length !== 0 && settings.retentionPoolMetadata !== 0) {
|
||||
promises.push(
|
||||
asyncMap(pools, async pool =>
|
||||
runTask(
|
||||
Task.run(
|
||||
{
|
||||
name: `Starting metadata backup for the pool (${pool.$id}). (${job.id})`,
|
||||
data: {
|
||||
properties: {
|
||||
id: pool.$id,
|
||||
name: `Starting metadata backup for the pool (${pool.$id}). (${job.id})`,
|
||||
pool,
|
||||
poolMaster: await ignoreErrors.call(pool.$xapi.getRecord('host', pool.master)),
|
||||
type: 'pool',
|
||||
@@ -100,17 +105,17 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings,
|
||||
}).run()
|
||||
)
|
||||
).catch(noop)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
if (job.xoMetadata !== undefined && settings.retentionXoMetadata !== 0) {
|
||||
promises.push(
|
||||
runTask(
|
||||
Task.run(
|
||||
{
|
||||
name: `Starting XO metadata backup. (${job.id})`,
|
||||
data: {
|
||||
properties: {
|
||||
name: `Starting XO metadata backup. (${job.id})`,
|
||||
type: 'xo',
|
||||
},
|
||||
},
|
||||
@@ -122,7 +127,7 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings,
|
||||
}).run()
|
||||
)
|
||||
).catch(noop)
|
||||
)
|
||||
}
|
||||
await Promise.all(promises)
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
|
||||
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
import { FullRemote } from './_vmRunners/FullRemote.mjs'
|
||||
import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -41,7 +42,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
const schedule = this._schedule
|
||||
const settings = this._settings
|
||||
|
||||
const throttleGenerator = new Throttle()
|
||||
const throttleGenerator = new Throttle(settings.maxExportRate)
|
||||
const throttleStream = createStreamThrottle(settings.maxExportRate)
|
||||
|
||||
const config = this._config
|
||||
|
||||
@@ -76,7 +78,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
}
|
||||
nTriesByVmId[vmUuid]++
|
||||
|
||||
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
|
||||
const taskStart = { properties: { id: vmUuid, name: 'backup VM', type: 'VM' } }
|
||||
const vmSettings = { ...settings, ...allSettings[vmUuid] }
|
||||
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
|
||||
|
||||
@@ -89,7 +91,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
schedule,
|
||||
settings: vmSettings,
|
||||
sourceRemoteAdapter,
|
||||
throttleGenerator,
|
||||
throttleGenerator, // for incrementals
|
||||
throttleStream, // for full
|
||||
vmUuid,
|
||||
}
|
||||
let vmBackup
|
||||
@@ -110,23 +113,27 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
||||
taskByVmId[vmUuid] = new Task(taskStart)
|
||||
}
|
||||
const task = taskByVmId[vmUuid]
|
||||
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
|
||||
let taskError
|
||||
return task
|
||||
.run(async () => {
|
||||
try {
|
||||
const result = await vmBackup.run()
|
||||
task.success(result)
|
||||
return result
|
||||
} catch (error) {
|
||||
if (isLastRun) {
|
||||
throw error
|
||||
} else {
|
||||
Task.warning(`Retry the VM mirror backup due to an error`, {
|
||||
attempt: nTriesByVmId[vmUuid],
|
||||
error: error.message,
|
||||
})
|
||||
queue.add(vmUuid)
|
||||
}
|
||||
.runInside(async () =>
|
||||
vmBackup.run().catch(error => {
|
||||
taskError = error
|
||||
})
|
||||
)
|
||||
.then(result => {
|
||||
if (taskError === undefined) {
|
||||
return task.success(result)
|
||||
}
|
||||
if (isLastRun) {
|
||||
return task.failure(taskError)
|
||||
}
|
||||
// don't end the task
|
||||
task.warning(`Retry the VM mirror backup due to an error`, {
|
||||
attempt: nTriesByVmId[vmUuid],
|
||||
error: taskError.message,
|
||||
})
|
||||
queue.add(vmUuid)
|
||||
})
|
||||
.catch(noop)
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
|
||||
import { runTask } from './_runTask.mjs'
|
||||
import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
|
||||
import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
|
||||
import { FullXapi } from './_vmRunners/FullXapi.mjs'
|
||||
import { Throttle } from '@vates/generator-toolbox'
|
||||
import createStreamThrottle from './_createStreamThrottle.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
@@ -55,7 +55,8 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
const schedule = this._schedule
|
||||
const settings = this._settings
|
||||
|
||||
const throttleGenerator = new Throttle()
|
||||
const throttleGenerator = new Throttle(settings.maxExportRate)
|
||||
const throttleStream = createStreamThrottle(settings.maxExportRate)
|
||||
|
||||
const config = this._config
|
||||
|
||||
@@ -63,13 +64,12 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
Disposable.all(
|
||||
extractIdsFromSimplePattern(job.srs).map(id =>
|
||||
this._getRecord('SR', id).catch(error => {
|
||||
runTask(
|
||||
Task.run(
|
||||
{
|
||||
name: 'get SR record',
|
||||
data: { type: 'SR', id },
|
||||
properties: { id, name: 'get SR record', type: 'SR' },
|
||||
},
|
||||
() => Promise.reject(error)
|
||||
)
|
||||
).catch(noop)
|
||||
})
|
||||
)
|
||||
),
|
||||
@@ -106,11 +106,12 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
}
|
||||
return taskByVmId[vmUuid]
|
||||
}
|
||||
const vmBackupFailed = error => {
|
||||
const vmBackupFailed = async (error, task) => {
|
||||
if (isLastRun) {
|
||||
throw error
|
||||
return task.failure(error)
|
||||
} else {
|
||||
Task.warning(`Retry the VM backup due to an error`, {
|
||||
// don't end the task
|
||||
task.warning(`Retry the VM backup due to an error`, {
|
||||
attempt: nTriesByVmId[vmUuid],
|
||||
error: error.message,
|
||||
})
|
||||
@@ -124,19 +125,21 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
nTriesByVmId[vmUuid]++
|
||||
|
||||
const vmSettings = { ...settings, ...allSettings[vmUuid] }
|
||||
const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
|
||||
const taskStart = { properties: { id: vmUuid, name: 'backup VM', type: 'VM' } }
|
||||
const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
|
||||
|
||||
return this._getRecord('VM', vmUuid).then(
|
||||
disposableVm =>
|
||||
Disposable.use(disposableVm, async vm => {
|
||||
if (taskStart.data.name_label === undefined) {
|
||||
taskStart.data.name_label = vm.name_label
|
||||
if (taskStart.properties.name_label === undefined) {
|
||||
taskStart.properties.name_label = vm.name_label
|
||||
}
|
||||
|
||||
const task = getVmTask()
|
||||
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
|
||||
let taskError
|
||||
return task
|
||||
.run(async () => {
|
||||
.runInside(async () => {
|
||||
const opts = {
|
||||
baseSettings,
|
||||
config,
|
||||
@@ -148,6 +151,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
settings: vmSettings,
|
||||
srs,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}
|
||||
|
||||
@@ -161,21 +165,21 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
||||
throw new Error(`Job mode ${job.mode} not implemented`)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await vmBackup.run()
|
||||
return vmBackup.run().catch(error => {
|
||||
taskError = error
|
||||
})
|
||||
})
|
||||
.then(result => {
|
||||
if (taskError === undefined) {
|
||||
task.success(result)
|
||||
return result
|
||||
} catch (error) {
|
||||
vmBackupFailed(error)
|
||||
} else {
|
||||
// ending the task with error or not ending the task
|
||||
vmBackupFailed(taskError, task)
|
||||
}
|
||||
})
|
||||
.catch(noop) // errors are handled by logs
|
||||
}),
|
||||
error =>
|
||||
getVmTask().run(() => {
|
||||
vmBackupFailed(error)
|
||||
})
|
||||
error => vmBackupFailed(error, getVmTask())
|
||||
)
|
||||
}
|
||||
const { concurrency } = settings
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import Disposable from 'promise-toolbox/Disposable'
|
||||
import pTimeout from 'promise-toolbox/timeout'
|
||||
import { compileTemplate } from '@xen-orchestra/template'
|
||||
import { runTask } from './_runTask.mjs'
|
||||
import { RemoteTimeoutError } from './_RemoteTimeoutError.mjs'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
export const DEFAULT_SETTINGS = {
|
||||
getRemoteTimeout: 300e3,
|
||||
@@ -36,13 +38,16 @@ export const Abstract = class AbstractRunner {
|
||||
})
|
||||
} catch (error) {
|
||||
// See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
|
||||
runTask(
|
||||
Task.run(
|
||||
{
|
||||
name: 'get remote adapter',
|
||||
data: { type: 'remote', id: remoteId },
|
||||
properties: {
|
||||
id: remoteId,
|
||||
name: 'get remote adapter',
|
||||
type: 'remote',
|
||||
},
|
||||
},
|
||||
() => Promise.reject(error)
|
||||
)
|
||||
).catch(noop)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { DIR_XO_POOL_METADATA_BACKUPS } from '../RemoteAdapter.mjs'
|
||||
import { forkStreamUnpipe } from './_forkStreamUnpipe.mjs'
|
||||
import { formatFilenameDate } from '../_filenameDate.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
|
||||
export const PATH_DB_DUMP = '/pool/xmldbdump'
|
||||
|
||||
@@ -54,8 +54,8 @@ export class PoolMetadataBackup {
|
||||
([remoteId, adapter]) =>
|
||||
Task.run(
|
||||
{
|
||||
name: `Starting metadata backup for the pool (${pool.$id}) for the remote (${remoteId}). (${job.id})`,
|
||||
data: {
|
||||
properties: {
|
||||
name: `Starting metadata backup for the pool (${pool.$id}) for the remote (${remoteId}). (${job.id})`,
|
||||
id: remoteId,
|
||||
type: 'remote',
|
||||
},
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { join } from '@xen-orchestra/fs/path'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { DIR_XO_CONFIG_BACKUPS } from '../RemoteAdapter.mjs'
|
||||
import { formatFilenameDate } from '../_filenameDate.mjs'
|
||||
import { Task } from '../Task.mjs'
|
||||
|
||||
export class XoMetadataBackup {
|
||||
constructor({ config, job, remoteAdapters, schedule, settings }) {
|
||||
@@ -51,8 +51,8 @@ export class XoMetadataBackup {
|
||||
([remoteId, adapter]) =>
|
||||
Task.run(
|
||||
{
|
||||
name: `Starting XO metadata backup for the remote (${remoteId}). (${job.id})`,
|
||||
data: {
|
||||
properties: {
|
||||
name: `Starting XO metadata backup for the remote (${remoteId}). (${job.id})`,
|
||||
id: remoteId,
|
||||
type: 'remote',
|
||||
},
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
import { Task } from '../Task.mjs'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
export const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
|
||||
@@ -2,10 +2,11 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
|
||||
import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
||||
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
|
||||
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
||||
|
||||
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
|
||||
_getRemoteWriter() {
|
||||
return FullRemoteWriter
|
||||
_getRemoteWriters() {
|
||||
return [FullRemoteWriter, AggregatedFullRemoteWriter]
|
||||
}
|
||||
|
||||
_filterTransferList(transferList) {
|
||||
@@ -16,7 +17,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
||||
const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
|
||||
|
||||
for (const metadata of transferList) {
|
||||
const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
|
||||
const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
|
||||
const sizeContainer = watchStreamSize(stream)
|
||||
|
||||
// @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
|
||||
|
||||
@@ -5,12 +5,14 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
||||
import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
|
||||
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
||||
import { AbstractXapi } from './_AbstractXapi.mjs'
|
||||
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
||||
import { AggregatedFullXapiWriter } from '../_writers/AggregatedFullXapiWriter.mjs'
|
||||
|
||||
const { debug } = createLogger('xo:backups:FullXapiVmBackup')
|
||||
|
||||
export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
_getWriters() {
|
||||
return [FullRemoteWriter, FullXapiWriter]
|
||||
return [FullRemoteWriter, FullXapiWriter, AggregatedFullRemoteWriter, AggregatedFullXapiWriter]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
@@ -49,16 +51,14 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
const { compression } = this.job
|
||||
const vm = this._vm
|
||||
const exportedVm = this._exportedVm
|
||||
// @todo put back throttle for full backup/Replication
|
||||
const stream =
|
||||
/* this._throttleStream( */
|
||||
const stream = this._throttleStream(
|
||||
(
|
||||
await this._xapi.VM_export(exportedVm.$ref, {
|
||||
compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
|
||||
useSnapshot: false,
|
||||
})
|
||||
).body
|
||||
/* ) */
|
||||
)
|
||||
|
||||
const vdis = await exportedVm.$getDisks()
|
||||
let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe
|
||||
@@ -66,9 +66,16 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
|
||||
const vdi = await this._xapi.getRecord('VDI', vdiRef)
|
||||
|
||||
// the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
|
||||
// of the real stream size in general, since a disk is never completely full
|
||||
// of the real stream size in general, since a disk is rarely completely full
|
||||
// vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
|
||||
maxStreamLength += vdi.virtual_size
|
||||
|
||||
// xva files are tar archive with data cut in 1MB blocks
|
||||
// each block is signed with a hash (in xxhash format)
|
||||
// each of these blocks + hash add one tar header entry in the file
|
||||
const XVA_BLOCK_SIZE = 1024 * 1024
|
||||
const nbBlocks = Math.ceil(vdi.virtual_size / XVA_BLOCK_SIZE)
|
||||
const headersSize = nbBlocks * (512 /* file header */ + 512 /* xxhash header */ + 512) /* xhxhash size */
|
||||
maxStreamLength += XVA_BLOCK_SIZE * nbBlocks + headersSize
|
||||
}
|
||||
|
||||
const sizeContainer = watchStreamSize(stream)
|
||||
|
||||
@@ -10,12 +10,13 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
|
||||
const { warn } = createLogger('xo:backups:Incrementalremote')
|
||||
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
_getRemoteWriter() {
|
||||
return IncrementalRemoteWriter
|
||||
_getRemoteWriters() {
|
||||
return [IncrementalRemoteWriter, AggregatedIncrementalRemoteWriter]
|
||||
}
|
||||
|
||||
// we'll transfer the full list if at least one backup should be transferred
|
||||
@@ -71,8 +72,9 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
||||
const isVhdDifferencing = {}
|
||||
|
||||
for (const key in incrementalExport.disks) {
|
||||
const disk = incrementalExport.disks[key]
|
||||
let disk = incrementalExport.disks[key]
|
||||
isVhdDifferencing[key] = disk.isDifferencing()
|
||||
disk = new ThrottledDisk(disk, this._throttleGenerator)
|
||||
incrementalExport.disks[key] = new SynchronizedDisk(disk)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { describe, it } from 'node:test'
|
||||
|
||||
import { IncrementalRemote } from './IncrementalRemote.mjs'
|
||||
|
||||
const makeAdapter = () => ({
|
||||
listVmBackups: async () => [],
|
||||
_handler: { getInfo: async () => ({ available: 0 }), _remote: { id: 'remote-id' } },
|
||||
})
|
||||
|
||||
const makeRunner = ({ distributeBackups = false, schedule } = {}) => {
|
||||
const remoteId = 'remote-id'
|
||||
return new IncrementalRemote({
|
||||
config: {},
|
||||
job: { id: 'job-id', mode: 'delta', remotes: { id: remoteId }, settings: {}, filter: undefined },
|
||||
healthCheckSr: undefined,
|
||||
remoteAdapters: { [remoteId]: makeAdapter() },
|
||||
schedule,
|
||||
settings: { distributeBackups, exportRetention: 1, deleteFirst: false, skipDeleteOldEntries: false },
|
||||
sourceRemoteAdapter: {},
|
||||
throttleGenerator: {},
|
||||
throttleStream: {},
|
||||
vmUuid: 'vm-uuid',
|
||||
})
|
||||
}
|
||||
|
||||
describe('IncrementalRemote prepare()', () => {
|
||||
it('prepare() with IncrementalRemoteWriter (distributeBackups: false)', async () => {
|
||||
const schedule = { id: 'schedule-id' }
|
||||
const runner = makeRunner({ distributeBackups: false, schedule })
|
||||
for (const writer of runner._writers) {
|
||||
await writer.prepare({ isFull: false })
|
||||
assert.equal(writer._schedule, schedule)
|
||||
}
|
||||
})
|
||||
|
||||
it('prepare() AggregatedIncrementalRemoteWriter (distributeBackups: true)', async () => {
|
||||
const schedule = { id: 'schedule-id' }
|
||||
const runner = makeRunner({ distributeBackups: true, schedule })
|
||||
const [aggWriter] = runner._writers
|
||||
await aggWriter.setupWriters()
|
||||
await aggWriter.prepare({ isFull: false })
|
||||
assert.equal(aggWriter.props.schedule, schedule)
|
||||
})
|
||||
})
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { Task } from '@vates/task'
|
||||
import keyBy from 'lodash/keyBy.js'
|
||||
|
||||
import { AbstractXapi } from './_AbstractXapi.mjs'
|
||||
@@ -6,7 +7,6 @@ import { forkDeltaExport } from './_forkDeltaExport.mjs'
|
||||
import { exportIncrementalVm } from '../../_incrementalVm.mjs'
|
||||
import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
|
||||
import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import {
|
||||
DATETIME,
|
||||
DELTA_CHAIN_LENGTH,
|
||||
@@ -14,13 +14,20 @@ import {
|
||||
setVmDeltaChainLength,
|
||||
markExportSuccessfull,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
||||
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
||||
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
|
||||
|
||||
export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
|
||||
_getWriters() {
|
||||
return [IncrementalRemoteWriter, IncrementalXapiWriter]
|
||||
return [
|
||||
IncrementalRemoteWriter,
|
||||
IncrementalXapiWriter,
|
||||
AggregatedIncrementalRemoteWriter,
|
||||
AggregatedIncrementalXapiWriter,
|
||||
]
|
||||
}
|
||||
|
||||
async _mustDoSnapshot() {
|
||||
@@ -43,13 +50,14 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
const isVhdDifferencing = {}
|
||||
let useNbd = false
|
||||
for (const key in deltaExport.disks) {
|
||||
const disk = deltaExport.disks[key]
|
||||
let disk = deltaExport.disks[key]
|
||||
isVhdDifferencing[key] = disk.isDifferencing()
|
||||
if (!isFull && !isVhdDifferencing[key] && key !== exportedVm.$suspend_VDI?.$ref) {
|
||||
Task.warning('Backup fell back to a full')
|
||||
}
|
||||
deltaExport.disks[key] = new SynchronizedDisk(disk)
|
||||
useNbd = useNbd || disk.useNbd()
|
||||
disk = new ThrottledDisk(disk, this._throttleGenerator)
|
||||
deltaExport.disks[key] = new SynchronizedDisk(disk)
|
||||
}
|
||||
if (useNbd) {
|
||||
Task.info('Transfer data using NBD')
|
||||
@@ -127,16 +135,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
const deltaChainLength = Math.max(
|
||||
...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
|
||||
)
|
||||
const fullInterval = this._settings.fullInterval
|
||||
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
|
||||
debug('not using base VM because fullInterval reached', {
|
||||
fullInterval,
|
||||
deltaChainLength,
|
||||
eq: fullInterval < deltaChainLength + 1,
|
||||
dc1: deltaChainLength + 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
|
||||
|
||||
@@ -165,6 +163,20 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
||||
debug('no base VM found')
|
||||
return
|
||||
}
|
||||
|
||||
// we do tafter checkbasevdis because we want the writer to know the target VM
|
||||
// especially on replication when we alwayr update the same VM
|
||||
const fullInterval = this._settings.fullInterval
|
||||
if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
|
||||
debug('not using base VM because fullInterval reached', {
|
||||
fullInterval,
|
||||
deltaChainLength,
|
||||
eq: fullInterval < deltaChainLength + 1,
|
||||
dc1: deltaChainLength + 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
baseUuidToSrcVdiUuid.forEach((srcVdiUuid, baseUuid) => {
|
||||
if (presentBaseVdis.has(baseUuid)) {
|
||||
debug('found base VDI', {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
const { debug, warn } = createLogger('xo:backups:AbstractVmRunner')
|
||||
|
||||
@@ -83,7 +83,7 @@ export const Abstract = class AbstractVmBackupRunner {
|
||||
// create a task to have an info in the logs and reports
|
||||
return Task.run(
|
||||
{
|
||||
name: 'health check',
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
() => {
|
||||
Task.info(`This VM doesn't match the health check's tags for this schedule`)
|
||||
|
||||
@@ -4,12 +4,12 @@ import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { defer } from 'golike-defer'
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
import { createPredicate } from 'value-matcher'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
|
||||
import { Abstract } from './_Abstract.mjs'
|
||||
import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
|
||||
export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
|
||||
_filterPredicate
|
||||
@@ -24,6 +24,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
settings,
|
||||
sourceRemoteAdapter,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vmUuid,
|
||||
}) {
|
||||
super()
|
||||
@@ -37,32 +38,48 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
this._healthCheckSr = healthCheckSr
|
||||
this._sourceRemoteAdapter = sourceRemoteAdapter
|
||||
this._throttleGenerator = throttleGenerator
|
||||
this._throttleStream = throttleStream
|
||||
this._vmUuid = vmUuid
|
||||
|
||||
const allSettings = job.settings
|
||||
const writers = new Set()
|
||||
this._writers = writers
|
||||
|
||||
const RemoteWriter = this._getRemoteWriter()
|
||||
extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
|
||||
const adapter = remoteAdapters[remoteId]
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
|
||||
if (settings.distributeBackups) {
|
||||
writers.add(
|
||||
new RemoteWriter({
|
||||
adapter,
|
||||
new AggregratedBackupWriter({
|
||||
adapters: remoteAdapters,
|
||||
BackupWriter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
schedule,
|
||||
vmUuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
})
|
||||
} else {
|
||||
extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
|
||||
const adapter = remoteAdapters[remoteId]
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
schedule,
|
||||
vmUuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
const { filter } = job
|
||||
if (filter === undefined) {
|
||||
this._filterPredicate = () => true
|
||||
@@ -72,28 +89,28 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
}
|
||||
|
||||
async #computeTransferListPerJob(sourceBackups, remotesBackups) {
|
||||
const localMetada = new Map()
|
||||
const localMetadata = new Map()
|
||||
sourceBackups.forEach(metadata => {
|
||||
const timestamp = metadata.timestamp
|
||||
localMetada.set(timestamp, metadata)
|
||||
localMetadata.set(timestamp, metadata)
|
||||
})
|
||||
const nbRemotes = remotesBackups.length
|
||||
const remoteMetadatas = {}
|
||||
const remoteMetadata = {}
|
||||
remotesBackups.forEach(async remoteBackups => {
|
||||
remoteBackups.forEach(metadata => {
|
||||
const timestamp = metadata.timestamp
|
||||
remoteMetadatas[timestamp] = (remoteMetadatas[timestamp] ?? 0) + 1
|
||||
remoteMetadata[timestamp] = (remoteMetadata[timestamp] ?? 0) + 1
|
||||
})
|
||||
})
|
||||
|
||||
let transferList = []
|
||||
const timestamps = [...localMetada.keys()]
|
||||
const timestamps = [...localMetadata.keys()]
|
||||
timestamps.sort()
|
||||
for (const timestamp of timestamps) {
|
||||
if (remoteMetadatas[timestamp] !== nbRemotes) {
|
||||
if (remoteMetadata[timestamp] !== nbRemotes) {
|
||||
// this backup is not present in all the remote
|
||||
// should be retransferred if not found later
|
||||
transferList.push(localMetada.get(timestamp))
|
||||
transferList.push(localMetadata.get(timestamp))
|
||||
} else {
|
||||
// backup is present in local and remote : the chain has already been transferred
|
||||
transferList = []
|
||||
@@ -101,17 +118,10 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
}
|
||||
if (transferList.length > 0) {
|
||||
const filteredTransferList = this._filterTransferList(transferList)
|
||||
if (filteredTransferList.length > 0) {
|
||||
return filteredTransferList
|
||||
} else {
|
||||
Task.info('This VM is excluded by the job filter')
|
||||
return []
|
||||
}
|
||||
return { transferList: filteredTransferList, filtered: true }
|
||||
} else {
|
||||
Task.info('No new data to upload for this VM')
|
||||
return { transferList, filtered: false }
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -125,12 +135,21 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
|
||||
Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
|
||||
)
|
||||
const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
|
||||
const transferByJobs = await Promise.all(
|
||||
const transferObjectsList = await Promise.all(
|
||||
Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
|
||||
this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
|
||||
)
|
||||
)
|
||||
return transferByJobs.flat(1)
|
||||
const transferList = transferObjectsList.map(transferObject => transferObject.transferList).flat(1)
|
||||
if (transferList.length === 0 && Object.values(sourceBackupByJobId).length > 0) {
|
||||
if (transferObjectsList.some(transferObject => transferObject.filtered)) {
|
||||
// Happens if any job transfer list wasn't empty before filter and is empty after filter.
|
||||
Task.info('This VM is excluded by the job filter')
|
||||
} else {
|
||||
Task.info('No new data to upload for this VM')
|
||||
}
|
||||
}
|
||||
return transferList
|
||||
}
|
||||
|
||||
async run($defer) {
|
||||
|
||||
@@ -1,22 +1,30 @@
|
||||
import assert from 'node:assert'
|
||||
import groupBy from 'lodash/groupBy.js'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { decorateMethodsWith } from '@vates/decorate-with'
|
||||
import { defer } from 'golike-defer'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import { Abstract } from './_Abstract.mjs'
|
||||
import {
|
||||
COPY_OF,
|
||||
DATETIME,
|
||||
JOB_ID,
|
||||
SCHEDULE_ID,
|
||||
populateVdisOtherConfig,
|
||||
VM_UUID,
|
||||
resetVmOtherConfig,
|
||||
setVmOtherConfig,
|
||||
setVmSnapshotContentKeys,
|
||||
} from '../../_otherConfig.mjs'
|
||||
|
||||
const { warn, info } = createLogger('xo:backups:AbstractXapi')
|
||||
|
||||
const TEMP_SNAPSHOT_NAME = 'xo-backup-temp-snapshot-name'
|
||||
|
||||
export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
constructor({
|
||||
config,
|
||||
@@ -29,6 +37,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
settings,
|
||||
srs,
|
||||
throttleGenerator,
|
||||
throttleStream,
|
||||
vm,
|
||||
}) {
|
||||
super()
|
||||
@@ -63,6 +72,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
this._jobId = job.id
|
||||
this._jobSnapshotVdis = undefined
|
||||
this._throttleGenerator = throttleGenerator
|
||||
this._throttleStream = throttleStream
|
||||
this._xapi = vm.$xapi
|
||||
|
||||
// Base VM for the export
|
||||
@@ -81,58 +91,92 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
const writers = new Set()
|
||||
this._writers = writers
|
||||
|
||||
const [BackupWriter, ReplicationWriter] = this._getWriters()
|
||||
const [BackupWriter, ReplicationWriter, AggregratedBackupWriter, AggregratedReplicationWriter] =
|
||||
this._getWriters()
|
||||
|
||||
const allSettings = job.settings
|
||||
Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
|
||||
if (settings.exportRetention > 0) {
|
||||
if (settings.distributeBackups) {
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
new AggregratedBackupWriter({
|
||||
adapters: remoteAdapters,
|
||||
BackupWriter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
schedule,
|
||||
vmUuid: vm.uuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
} else {
|
||||
Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
writers.add(
|
||||
new BackupWriter({
|
||||
adapter,
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
schedule,
|
||||
vmUuid: vm.uuid,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
srs.forEach(sr => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[sr.uuid],
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
}
|
||||
if (settings.copyRetention) {
|
||||
if (settings.distributeReplications) {
|
||||
writers.add(
|
||||
new ReplicationWriter({
|
||||
new AggregratedReplicationWriter({
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
scheduleId: schedule.id,
|
||||
ReplicationWriter,
|
||||
schedule,
|
||||
vmUuid: vm.uuid,
|
||||
sr,
|
||||
settings: targetSettings,
|
||||
srs,
|
||||
settings,
|
||||
})
|
||||
)
|
||||
} else {
|
||||
srs.forEach(sr => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[sr.uuid],
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
writers.add(
|
||||
new ReplicationWriter({
|
||||
config,
|
||||
healthCheckSr,
|
||||
job,
|
||||
schedule,
|
||||
vmUuid: vm.uuid,
|
||||
sr,
|
||||
settings: targetSettings,
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensure the VM itself does not have any backup metadata which would be
|
||||
// copied on manual snapshots and interfere with the backup jobs
|
||||
|
||||
async _cleanMetadata() {
|
||||
const vm = this._vm
|
||||
if (JOB_ID in vm.other_config) {
|
||||
await resetVmOtherConfig(this._xapi, vm.$ref)
|
||||
}
|
||||
await resetVmOtherConfig(this._xapi, this._vm.$ref)
|
||||
}
|
||||
|
||||
async _snapshot() {
|
||||
@@ -142,7 +186,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
const settings = this._settings
|
||||
|
||||
if (await this._mustDoSnapshot()) {
|
||||
await Task.run({ name: 'snapshot' }, async () => {
|
||||
await Task.run({ properties: { name: 'snapshot' } }, async () => {
|
||||
if (!settings.bypassVdiChainsCheck) {
|
||||
await vm.$assertHealthyVdiChains()
|
||||
}
|
||||
@@ -158,7 +202,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
|
||||
const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
|
||||
ignoredVdisTag: '[NOBAK]',
|
||||
name_label: this._getSnapshotNameLabel(vm),
|
||||
name_label: TEMP_SNAPSHOT_NAME,
|
||||
unplugVusbs: true,
|
||||
})
|
||||
this.timestamp = Date.now()
|
||||
@@ -168,6 +212,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
scheduleId: this.scheduleId,
|
||||
vmUuid: vm.uuid,
|
||||
})
|
||||
await setVmSnapshotContentKeys(xapi, snapshotRef)
|
||||
const snapshot = await xapi.getRecord('VM', snapshotRef)
|
||||
await snapshot.set_name_label(this._getSnapshotNameLabel(vm))
|
||||
// reload data to ensure it is up to date with the new name label
|
||||
this._exportedVm = await xapi.getRecord('VM', snapshotRef)
|
||||
return this._exportedVm.uuid
|
||||
})
|
||||
@@ -177,43 +225,119 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
}
|
||||
|
||||
// handle snapshot by VDI since snapshot of cbtDestroySnapshotData jobs
|
||||
// aren't attached to any VM
|
||||
|
||||
// look at all the vdi snapshots ( snapshot_of not empty )
|
||||
// with the same vm_uuid and job_uuid
|
||||
// for cbt_metadata list them unconditionnaly to remove older one
|
||||
// for other: only list them if they are attached to a VM snapshot
|
||||
// and if this vm snapshot is also part of the backup
|
||||
// ensure they are attached to only one vm snapshot
|
||||
// ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
|
||||
|
||||
async _fetchJobSnapshots() {
|
||||
const jobId = this._jobId
|
||||
const vmRef = this._vm.$ref
|
||||
const xapi = this._xapi
|
||||
|
||||
// to ensure compatibility with snapshots older than CBT implementation
|
||||
// update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
|
||||
// remove by then end of 2024
|
||||
const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
|
||||
const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
|
||||
const vdiCandidates = {}
|
||||
const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
|
||||
Object.values(xapi.objects.indexes.type.VDI)
|
||||
.filter(_ => !!_) // filter nullish
|
||||
.filter(({ other_config, snapshot_of }) => {
|
||||
return (
|
||||
vdiUuids.includes(snapshot_of) &&
|
||||
other_config[JOB_ID] === jobId &&
|
||||
other_config[VM_UUID] === this._vm.uuid &&
|
||||
other_config[COPY_OF] === undefined
|
||||
)
|
||||
})
|
||||
.forEach(vdi => {
|
||||
vdiCandidates[vdi.uuid] = vdi
|
||||
})
|
||||
|
||||
const vmSnapshots = []
|
||||
vmSnapshotsOtherConfig.forEach((other_config, i) => {
|
||||
if (other_config[JOB_ID] === jobId) {
|
||||
vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
|
||||
}
|
||||
})
|
||||
await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
|
||||
// end of compatibility handling
|
||||
// check that user snapshots are clean
|
||||
|
||||
// handle snapshot by VDI
|
||||
this._jobSnapshotVdis = []
|
||||
const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
|
||||
for (const srcVdi of srcVdis) {
|
||||
const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
|
||||
for (const snapshot of snapshots) {
|
||||
// only keep the snapshot related to this backup job
|
||||
// and only if the job is still using purge snapshot data or if the disk
|
||||
// is not a cbt metadata disk ( expect a type: user for normal disks)
|
||||
if (
|
||||
snapshot.other_config[JOB_ID] === jobId &&
|
||||
(this._settings.cbtDestroySnapshotData || snapshot.type !== 'cbt_metadata')
|
||||
) {
|
||||
this._jobSnapshotVdis.push(snapshot)
|
||||
}
|
||||
for (const vdi of Object.values(vdiCandidates)) {
|
||||
// cbt metadata are always considered linked to a backup job
|
||||
// if they have the right other_config
|
||||
if (vdi.type === 'cbt_metadata') {
|
||||
continue
|
||||
}
|
||||
const vbds = vdi.$VBDs
|
||||
.filter(({ $VM }) => !!$VM) // filter empty VMs
|
||||
.filter(({ $VM }) => $VM.is_control_domain === false) // do not handle control domain
|
||||
if (vbds.length === 0) {
|
||||
// orphan vdi snapshot
|
||||
info(
|
||||
`disk snapshot ${vdi.name_label} is orphan or attached only to control domain,
|
||||
it will be removed at the end of a successful backup run`,
|
||||
{ vdi, attachedto: vdi.$VBDs.map(vbd => vbd?.$VM) }
|
||||
)
|
||||
continue
|
||||
}
|
||||
const userVms = vbds.map(({ $VM }) => $VM)
|
||||
if (vbds.length > 1) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms : ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, userVms }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
const vm = vbds[0].$VM
|
||||
|
||||
// xapi has some issue regarding vdi snapshot attached to non snapshot VM
|
||||
// it can also be some user action forcibely linking a vdi snapshot to a vm
|
||||
// => we exclude these from the backup processing
|
||||
if (vm.$snapshot_of === undefined) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
// vdi is attached only to a snapshot that is not a backup snapshot
|
||||
// we don't check scheduleId since we are looking for all the snapshot of this job
|
||||
// => excludes from the list to be cleared
|
||||
if (
|
||||
vm.other_config[DATETIME] !== vdi.other_config[DATETIME] ||
|
||||
vm.other_config[JOB_ID] !== vdi.other_config[JOB_ID] ||
|
||||
vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
|
||||
) {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
|
||||
This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm }
|
||||
)
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
continue
|
||||
}
|
||||
|
||||
// check if all the disks of these VM snapshot have been harvested
|
||||
// if not => remove it from the list to ensure we won't half destroy VM later
|
||||
vm.$VBDs
|
||||
.filter(({ $VDI }) => !!$VDI) // filter missing keys
|
||||
.filter(({ $VDI }) => $VDI.$snapshot_of !== undefined) // skip non-snapshot VDIs (e.g., ISOs/CD-ROMs)
|
||||
.filter(({ $VDI }) => $VDI && vdiCandidates[$VDI.uuid] === undefined)
|
||||
.forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
|
||||
warn(
|
||||
`vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
|
||||
linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
|
||||
is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
|
||||
{ vdi, vm, vbds, outOfSnapshotsVdi }
|
||||
)
|
||||
// this will be called multiple time but it is not really an issue
|
||||
delete vdiCandidates[vdi.uuid]
|
||||
})
|
||||
}
|
||||
|
||||
this._jobSnapshotVdis = Object.values(vdiCandidates)
|
||||
}
|
||||
|
||||
async _removeUnusedSnapshots() {
|
||||
@@ -253,28 +377,47 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
return
|
||||
}
|
||||
const vdis = snapshotPerDatetime[datetime]
|
||||
let vmRef
|
||||
let vm
|
||||
// if there is an attached VM => destroy the VM (Non CBT backups)
|
||||
for (const vdi of vdis) {
|
||||
const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
|
||||
if (vbds.length > 0) {
|
||||
// only one VM linked to this vdi
|
||||
// this will throw error for VDI still attached to control domain
|
||||
// since we won't be able to remove an attached VDI
|
||||
assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
|
||||
const vm = vbds[0].$VM
|
||||
assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
|
||||
|
||||
const vmRefVdi = vm.$ref
|
||||
// same vm than other vdi of the same batch
|
||||
assert.ok(
|
||||
vmRef === undefined || vmRef === vmRefVdi,
|
||||
'_removeUnusedSnapshots don t handle vdi related to multiple VMs '
|
||||
)
|
||||
vmRef = vmRefVdi
|
||||
const vdiVm = vbds[0].$VM
|
||||
if (vdiVm.$snapshot_of === undefined) {
|
||||
// don't delete a VM (especially a control domain)
|
||||
warn(
|
||||
`VM ${vdiVm.uuid} (${vdiVm.name_label}) linked to VDI ${vdi.uuid} (${vdi.name_label}) should be a snapshot`
|
||||
)
|
||||
throw new Error(`VM must be a snapshot`)
|
||||
}
|
||||
if (vm !== undefined && vm.$ref !== vdiVm.$ref) {
|
||||
// this VDI is attached to another VM than the other vdi of
|
||||
// this batch
|
||||
// in doubt, do not delete anything
|
||||
warn("_removeUnusedSnapshots don't handle vdi related to multiple VMs ", {
|
||||
vm1: {
|
||||
label: vm.name_label,
|
||||
id: vm.$id,
|
||||
},
|
||||
vm2: {
|
||||
label: vdiVm.name_label,
|
||||
id: vdiVm.$id,
|
||||
},
|
||||
vdis: vdis.map(({ name_label, $id }) => ({ name_label, $id })),
|
||||
})
|
||||
throw new Error(
|
||||
`_removeUnusedSnapshots don't handle vdi related to multiple VMs ${vm.name_label} and ${vdiVm.name_label}`
|
||||
)
|
||||
}
|
||||
vm = vdiVm
|
||||
}
|
||||
}
|
||||
if (vmRef !== undefined) {
|
||||
return xapi.VM_destroy(vmRef)
|
||||
if (vm?.$ref !== undefined) {
|
||||
return xapi.VM_destroy(vm.$ref)
|
||||
} else {
|
||||
return asyncMap(
|
||||
vdis.map(async ({ $ref }) => {
|
||||
@@ -284,6 +427,11 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// list and remove the snapshot were the jobs failed between
|
||||
// makesnapshot and update_other_config
|
||||
const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)
|
||||
await asyncEach(snapshots, snapshot => snapshot.$destroy())
|
||||
}
|
||||
|
||||
async _removeSnapshotData() {
|
||||
@@ -291,8 +439,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
|
||||
// going back to a previous version of XO not supporting CBT will create a full backup
|
||||
// this will only do something after snapshot and transfer
|
||||
if (
|
||||
this._exportedVm !== undefined &&
|
||||
// don't modify the VM
|
||||
this._exportedVm?.is_a_snapshot &&
|
||||
this._exportedVm.$snapshot_of !== undefined &&
|
||||
// user don't want to keep the snapshot data
|
||||
this._settings.snapshotRetention === 0 &&
|
||||
// preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
|
||||
|
||||
export class AggregatedFullRemoteWriter extends AbstractAggregatedRemoteWriter {
|
||||
async deleteOldBackupsOnAdapter(adapter, backups) {
|
||||
await adapter.deleteFullVmBackups(backups)
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
|
||||
await this.setOldBackups()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
async run(props) {
|
||||
const mainAdapter = await this.computeAdapterCandidate()
|
||||
this.mainWriter = this.getWriterByAdapter(mainAdapter)
|
||||
return this.mainWriter.run(props)
|
||||
}
|
||||
|
||||
// cleanup if deleteFirst is NOT selected
|
||||
// release the lock on every adapters with this VM
|
||||
async afterBackup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
|
||||
|
||||
export class AggregatedFullXapiWriter extends AbstractAggregatedXapiWriter {
|
||||
// delete replica if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
this.setOldReplicaList()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
}
|
||||
|
||||
async run(props) {
|
||||
const mainSr = await this.computeSRCandidate()
|
||||
this.mainWriter = this.getWriterBySr(mainSr)
|
||||
await this.mainWriter.run(props)
|
||||
}
|
||||
|
||||
// delete replica if not using deleteFirst
|
||||
async afterBackup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
|
||||
export class AggregatedIncrementalRemoteWriter extends AbstractAggregatedRemoteWriter {
|
||||
async deleteOldBackupsOnAdapter(adapter, backups) {
|
||||
await adapter.deleteDeltaVmBackups(backups)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Map<string,string>} baseUuidToSrcVdi
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
let selectedCopy = new Map()
|
||||
for (const writer of this.writers) {
|
||||
const copy = new Map(baseUuidToSrcVdi)
|
||||
await writer.checkBaseVdis(copy)
|
||||
if (copy.size > 0) {
|
||||
// there can be multiple candidates
|
||||
// we use the last one in this case
|
||||
this.mainWriter = writer
|
||||
selectedCopy = copy
|
||||
}
|
||||
}
|
||||
// update the parameter with our best candidate
|
||||
// that can be an empty map
|
||||
baseUuidToSrcVdi.forEach(key => {
|
||||
if (!selectedCopy.has(key)) {
|
||||
baseUuidToSrcVdi.delete(key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
|
||||
await this.setOldBackups()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
async prepare({ isFull }) {
|
||||
if (this.mainWriter === undefined) {
|
||||
// use the remote with the most empty space for starting a new chain
|
||||
const mainAdapter = await this.computeAdapterCandidate()
|
||||
this.mainWriter = this.getWriterByAdapter(mainAdapter)
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.prepare({ isFull })))
|
||||
}
|
||||
// write only on the main writer
|
||||
transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
|
||||
return this.mainWriter.transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot })
|
||||
}
|
||||
|
||||
// chain only on the main writer
|
||||
updateUuidAndChain({ isVhdDifferencing, vdis }) {
|
||||
return this.mainWriter.updateUuidAndChain({ isVhdDifferencing, vdis })
|
||||
}
|
||||
|
||||
// remove the backups and remove the entries
|
||||
async cleanup() {
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldBackups()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.cleanup()))
|
||||
}
|
||||
// cleanVM on all, since we may want to delete a chani
|
||||
// that is not on the main writer
|
||||
async afterBackup() {
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AggregatedIncrementalXapiWriter')
|
||||
export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWriter {
|
||||
/**
|
||||
*
|
||||
* @param {Map<string,string>} baseUuidToSrcVdi
|
||||
*/
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
debug('checkBaseVdis', { baseUuidToSrcVdi })
|
||||
let selectedCopy = new Map()
|
||||
for (const writer of this.writers) {
|
||||
const copy = new Map(baseUuidToSrcVdi)
|
||||
await writer.checkBaseVdis(copy)
|
||||
if (copy.size > 0) {
|
||||
debug('checkBaseVdis found a mainwriter candidate ', writer._sr.name_label)
|
||||
// there can be multiple candidates
|
||||
// we use the last one in this case
|
||||
this.mainWriter = writer
|
||||
selectedCopy = copy
|
||||
} else {
|
||||
debug(writer._sr.name_label, 'is not a delta candidate')
|
||||
}
|
||||
}
|
||||
// update the parameter with our best candidate
|
||||
// that can be an empty map
|
||||
baseUuidToSrcVdi.forEach(key => {
|
||||
if (!selectedCopy.has(key)) {
|
||||
baseUuidToSrcVdi.delete(key)
|
||||
}
|
||||
})
|
||||
debug('checkBaseVdis', { baseUuidToSrcVdi })
|
||||
}
|
||||
|
||||
// take the lock on every adapters with this VM
|
||||
// cleanup if deleteFirst is selected
|
||||
async beforeBackup() {
|
||||
await this.setupWriters()
|
||||
await Promise.all(this.writers.map(writer => writer.beforeBackup()))
|
||||
this.setOldReplicaList()
|
||||
if (this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
}
|
||||
|
||||
async prepare(args) {
|
||||
if (this.mainWriter === undefined) {
|
||||
debug('no mainwriter found, fallback to a new SR')
|
||||
// use the SR with the most empty space for starting a new chain
|
||||
const mainSr = await this.computeSRCandidate()
|
||||
this.mainWriter = this.getWriterBySr(mainSr)
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.prepare(args)))
|
||||
}
|
||||
|
||||
// write only on the main writer
|
||||
transfer(args) {
|
||||
return this.mainWriter.transfer(args)
|
||||
}
|
||||
|
||||
// chain only on the main writer
|
||||
updateUuidAndChain(args) {
|
||||
return this.mainWriter.updateUuidAndChain(args)
|
||||
}
|
||||
|
||||
// remove the backups and remove the entries
|
||||
async cleanup() {
|
||||
debug('cleanup')
|
||||
if (!this.props.settings.deleteFirst) {
|
||||
await this.deleteOldReplicas()
|
||||
}
|
||||
await Promise.all(this.writers.map(writer => writer.cleanup()))
|
||||
}
|
||||
|
||||
// cleanVM on all, since we may want to delete a chani
|
||||
// that is not on the main writer
|
||||
async afterBackup() {
|
||||
await Promise.all(this.writers.map(writer => writer.afterBackup()))
|
||||
}
|
||||
|
||||
healthCheck() {
|
||||
return this.mainWriter.healthCheck()
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
|
||||
import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
|
||||
import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
|
||||
@@ -9,11 +10,11 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
|
||||
this.run = Task.wrapFn(
|
||||
this.run = Task.wrap(
|
||||
{
|
||||
name: 'export',
|
||||
data: {
|
||||
properties: {
|
||||
id: props.remoteId,
|
||||
name: 'export',
|
||||
type: 'remote',
|
||||
|
||||
// necessary?
|
||||
@@ -27,13 +28,13 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
|
||||
const settings = this._settings
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
const scheduleId = this._schedule.id
|
||||
|
||||
const adapter = this._adapter
|
||||
let metadata = await this._isAlreadyTransferred(timestamp)
|
||||
if (metadata !== undefined) {
|
||||
// @todo : should skip backup while being vigilant to not stuck the forked stream
|
||||
Task.info('This backup has already been transfered')
|
||||
Task.info('This backup has already been transferred')
|
||||
}
|
||||
|
||||
const oldBackups = getOldEntries(
|
||||
@@ -59,12 +60,12 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
xva: './' + dataBasename,
|
||||
}
|
||||
|
||||
const { deleteFirst } = settings
|
||||
if (deleteFirst) {
|
||||
const { skipDeleteOldEntries, deleteFirst } = settings
|
||||
if (!skipDeleteOldEntries && deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
await Task.run({ properties: { name: 'transfer' } }, async () => {
|
||||
await adapter.outputStream(dataFilename, stream, {
|
||||
maxStreamLength,
|
||||
streamLength,
|
||||
@@ -73,9 +74,10 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
|
||||
return { size: sizeContainer.size }
|
||||
})
|
||||
metadata.size = sizeContainer.size
|
||||
metadata.tags = await this.getLongTermRetentionTags(metadata)
|
||||
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
|
||||
|
||||
if (!deleteFirst) {
|
||||
if (!skipDeleteOldEntries && !deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
|
||||
import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
|
||||
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
|
||||
@@ -14,11 +14,11 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
|
||||
this.run = Task.wrapFn(
|
||||
this.run = Task.wrap(
|
||||
{
|
||||
name: 'export',
|
||||
data: {
|
||||
properties: {
|
||||
id: props.sr.uuid,
|
||||
name: 'export',
|
||||
name_label: this._sr.name_label,
|
||||
type: 'SR',
|
||||
|
||||
@@ -34,7 +34,7 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
const sr = this._sr
|
||||
const settings = this._settings
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
const scheduleId = this._schedule.id
|
||||
|
||||
const { uuid: srUuid, $xapi: xapi } = sr
|
||||
|
||||
@@ -46,13 +46,13 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
const oldVms = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
|
||||
|
||||
const deleteOldBackups = () => asyncMapSettled(oldVms, vm => xapi.VM_destroy(vm.$ref))
|
||||
const { deleteFirst, _warmMigration } = settings
|
||||
if (deleteFirst) {
|
||||
const { deleteFirst, skipDeleteOldEntries, _warmMigration } = settings
|
||||
if (!skipDeleteOldEntries && deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
|
||||
let targetVmRef
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
await Task.run({ properties: { name: 'transfer' } }, async () => {
|
||||
targetVmRef = await xapi.VM_import(stream, sr.$ref, vm =>
|
||||
Promise.all([
|
||||
!_warmMigration && vm.add_tags('Disaster Recovery'),
|
||||
@@ -85,7 +85,7 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
|
||||
}),
|
||||
])
|
||||
|
||||
if (!deleteFirst) {
|
||||
if (!skipDeleteOldEntries && !deleteFirst) {
|
||||
await deleteOldBackups()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,10 @@ import { createLogger } from '@xen-orchestra/log'
|
||||
import { decorateClass } from '@vates/decorate-with'
|
||||
import { defer } from 'golike-defer'
|
||||
import { dirname, basename } from 'node:path'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
|
||||
import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
|
||||
import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
|
||||
@@ -72,25 +72,26 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
prepare({ isFull }) {
|
||||
// create the task related to this export and ensure all methods are called in this context
|
||||
const task = new Task({
|
||||
name: 'export',
|
||||
data: {
|
||||
properties: {
|
||||
id: this._remoteId,
|
||||
isFull,
|
||||
name: 'export',
|
||||
type: 'remote',
|
||||
},
|
||||
})
|
||||
this.transfer = task.wrapFn(this.transfer)
|
||||
this.healthCheck = task.wrapFn(this.healthCheck)
|
||||
this.cleanup = task.wrapFn(this.cleanup)
|
||||
this.afterBackup = task.wrapFn(this.afterBackup, true)
|
||||
this._prepare = task.wrapInside(this._prepare)
|
||||
this.transfer = task.wrapInside(this.transfer)
|
||||
this.healthCheck = task.wrapInside(this.healthCheck)
|
||||
this.cleanup = task.wrapInside(this.cleanup)
|
||||
this.afterBackup = task.wrap(this.afterBackup)
|
||||
|
||||
return task.run(() => this._prepare())
|
||||
return this._prepare()
|
||||
}
|
||||
|
||||
async _prepare() {
|
||||
const adapter = this._adapter
|
||||
const settings = this._settings
|
||||
const scheduleId = this._scheduleId
|
||||
const scheduleId = this._schedule.id
|
||||
const vmUuid = this._vmUuid
|
||||
|
||||
const oldEntries = getOldEntries(
|
||||
@@ -114,13 +115,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
oldEntries.length = maxMergedDeltasPerRun
|
||||
}
|
||||
|
||||
if (settings.deleteFirst) {
|
||||
if (settings.deleteFirst && !settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
if (!this._settings.deleteFirst) {
|
||||
if (!this._settings.deleteFirst && !this._settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
@@ -180,7 +181,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
async _transfer($defer, { isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
|
||||
const adapter = this._adapter
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
const scheduleId = this._schedule.id
|
||||
const settings = this._settings
|
||||
const jobId = job.id
|
||||
const handler = adapter.handler
|
||||
@@ -223,7 +224,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
vtpms: deltaExport.vtpms,
|
||||
}
|
||||
let size = 0
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
await Task.run({ properties: { name: 'transfer' } }, async () => {
|
||||
await asyncEach(
|
||||
Object.entries(deltaExport.disks),
|
||||
async ([diskRef, disk]) => {
|
||||
@@ -244,6 +245,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
|
||||
|
||||
return { size }
|
||||
})
|
||||
metadataContent.tags = await this.getLongTermRetentionTags(metadataContent)
|
||||
metadataContent.size = size // @todo return exactly the size written by this writer
|
||||
this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
|
||||
|
||||
|
||||
@@ -1,79 +1,227 @@
|
||||
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import humanFormat from 'human-format'
|
||||
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
import { Task } from '@vates/task'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { importIncrementalVm } from '../../_incrementalVm.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
|
||||
import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
|
||||
import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
|
||||
import { listReplicatedVms } from './_listReplicatedVms.mjs'
|
||||
import { COPY_OF, setVmOtherConfig, BASE_DELTA_VDI } from '../../_otherConfig.mjs'
|
||||
import { compareReplicatedVmDatetime, listReplicatedVms } from './_listReplicatedVms.mjs'
|
||||
import {
|
||||
COPY_OF,
|
||||
setVmOtherConfig,
|
||||
BASE_DELTA_VDI,
|
||||
JOB_ID,
|
||||
SCHEDULE_ID,
|
||||
REPLICATED_TO_SR_UUID,
|
||||
DATETIME,
|
||||
VM_UUID,
|
||||
} from '../../_otherConfig.mjs'
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { XapiDiskSource } from '@xen-orchestra/xapi'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug } = createLogger('xo:backups:IncrementalXapiWriter')
|
||||
|
||||
export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
|
||||
// Map of source VDI UUID (COPY_OF) → validated active VDI on the target SR.
|
||||
// Built by checkBaseVdis, consumed by #decorateVmMetadata to set baseVdi.
|
||||
#baseVdisBySourceUuid = new Map()
|
||||
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
const sr = this._sr
|
||||
this.#baseVdisBySourceUuid = new Map()
|
||||
|
||||
if (baseUuidToSrcVdi.size === 0) {
|
||||
// searching for the vdis is expensive
|
||||
// don't do it if there is nothing to find
|
||||
return
|
||||
}
|
||||
|
||||
// @todo use an index if possible
|
||||
// @todo : this seems similar to decorateVmMetadata
|
||||
const replicatedVdis = sr.$VDIs
|
||||
.filter(vdi => {
|
||||
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
|
||||
return (
|
||||
vdi?.managed &&
|
||||
!vdi?.is_a_snapshot /* only look for real vdi */ &&
|
||||
baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
|
||||
)
|
||||
// look for the same snapshot
|
||||
// ensure there are no data between the snapshot and the active disk
|
||||
|
||||
const snapshotCandidates = sr.$VDIs.filter(vdi => {
|
||||
return (
|
||||
vdi?.managed &&
|
||||
vdi?.is_a_snapshot &&
|
||||
vdi.other_config[JOB_ID] === this._job.id &&
|
||||
vdi.other_config[VM_UUID] === this._vmUuid &&
|
||||
baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
|
||||
)
|
||||
})
|
||||
debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.length)
|
||||
|
||||
if (snapshotCandidates.length > 0) {
|
||||
// reset before searching for candidates
|
||||
this.#baseVdisBySourceUuid = new Map()
|
||||
this._targetVmRef = undefined
|
||||
const { baseVdisBySourceUuid, targetVmRef } = await this.#validateSnapshotCandidates(snapshotCandidates)
|
||||
for (const [sourceUuid, vdi] of baseVdisBySourceUuid) {
|
||||
this.#baseVdisBySourceUuid.set(sourceUuid, vdi)
|
||||
}
|
||||
if (targetVmRef !== undefined) {
|
||||
this._targetVmRef = targetVmRef
|
||||
}
|
||||
} else {
|
||||
// Legacy fallback (upgrade from pre-6.3): no target snapshots exist yet,
|
||||
// look for active (non-snapshot) VDIs with matching COPY_OF, like the old code did.
|
||||
debug('checkBaseVdis, no snapshot candidates, falling back to legacy active VDI lookup')
|
||||
const legacyVdis = sr.$VDIs.filter(vdi => {
|
||||
return vdi?.managed && !vdi?.is_a_snapshot && baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
|
||||
})
|
||||
.map(({ other_config }) => other_config?.[COPY_OF])
|
||||
.filter(_ => !!_)
|
||||
for (const vdi of legacyVdis) {
|
||||
const sourceUuid = vdi.other_config[COPY_OF]
|
||||
if (sourceUuid) {
|
||||
this.#baseVdisBySourceUuid.set(sourceUuid, vdi)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const uuid of baseUuidToSrcVdi.keys()) {
|
||||
if (!replicatedVdis.includes(uuid)) {
|
||||
if (!this.#baseVdisBySourceUuid.has(uuid)) {
|
||||
baseUuidToSrcVdi.delete(uuid)
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 6.3+ snapshot-based validation: for each snapshot candidate, check whether
|
||||
* the active VDI has diverged from the snapshot. Returns a baseVdisBySourceUuid
|
||||
* map and, when all disks are clean, the targetVmRef to reuse.
|
||||
*/
|
||||
async #validateSnapshotCandidates(snapshotCandidates) {
|
||||
const sr = this._sr
|
||||
const baseVdisBySourceUuid = new Map()
|
||||
let targetVmRef
|
||||
let canChainToTargetVm = true
|
||||
|
||||
await asyncEach(
|
||||
snapshotCandidates,
|
||||
async snapshot => {
|
||||
let diffDisk
|
||||
let activeVdi
|
||||
try {
|
||||
activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
|
||||
const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
|
||||
if (userVbds.length !== 1) {
|
||||
debug('checkBaseVdis, shared vbd ', { ref: snapshot.$ref, userVbds })
|
||||
// shared vdi ignore
|
||||
return
|
||||
}
|
||||
const vm = userVbds[0].$VM
|
||||
if (!('start' in vm.blocked_operations)) {
|
||||
debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
|
||||
// vm start unlocked
|
||||
// not really an issue since we have check the delta
|
||||
// but it indicates the users played with the blocked operations
|
||||
return
|
||||
}
|
||||
diffDisk = new XapiDiskSource({
|
||||
xapi: sr.$xapi,
|
||||
vdiRef: activeVdi.$ref,
|
||||
baseRef: snapshot.$ref,
|
||||
onlyListChangedBlocks: true,
|
||||
})
|
||||
await diffDisk.init()
|
||||
const sourceUuid = snapshot.other_config?.[COPY_OF]
|
||||
if (diffDisk.getBlockIndexes().length === 0) {
|
||||
// no block modification since the common snapshot, we can chain VM and disk
|
||||
if (sourceUuid) {
|
||||
baseVdisBySourceUuid.set(sourceUuid, activeVdi)
|
||||
}
|
||||
// Track the target VM (the replicated VM to update on the next transfer).
|
||||
targetVmRef = vm.$ref
|
||||
} else {
|
||||
if (sourceUuid) {
|
||||
baseVdisBySourceUuid.set(sourceUuid, snapshot)
|
||||
}
|
||||
// there are changed block since the snapshot
|
||||
// we can reuse it to transfer a delta, but we will
|
||||
// create a new VM
|
||||
canChainToTargetVm = false
|
||||
debug('checkBaseVdis, data between snapshot and active disk', {
|
||||
vdiRef: snapshot.$ref,
|
||||
nbBlocks: diffDisk.getBlockIndexes().length,
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
debug('checkBaseVdis, skipping snapshot', { ref: snapshot.$ref, error })
|
||||
return
|
||||
} finally {
|
||||
await diffDisk?.close().catch(error => debug('checkBaseVdis, error closing', error))
|
||||
await sr.$xapi.VDI_disconnectFromControlDomain(snapshot.$ref)
|
||||
if (activeVdi !== undefined) {
|
||||
await sr.$xapi.VDI_disconnectFromControlDomain(activeVdi.$ref)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
concurrency: 4,
|
||||
}
|
||||
)
|
||||
|
||||
if (!canChainToTargetVm) {
|
||||
// if at least one disk has new data, create a new VM
|
||||
// instead of updating it
|
||||
targetVmRef = undefined
|
||||
} else if (targetVmRef !== undefined) {
|
||||
debug('checkBaseVdis,got a valid vm target', targetVmRef)
|
||||
}
|
||||
|
||||
return { baseVdisBySourceUuid, targetVmRef }
|
||||
}
|
||||
|
||||
updateUuidAndChain() {
|
||||
// nothing to do, the chaining is not modified in this case
|
||||
}
|
||||
prepare({ isFull }) {
|
||||
// create the task related to this export and ensure all methods are called in this context
|
||||
const task = new Task({
|
||||
name: 'export',
|
||||
data: {
|
||||
properties: {
|
||||
id: this._sr.uuid,
|
||||
isFull,
|
||||
name: 'export',
|
||||
name_label: this._sr.name_label,
|
||||
type: 'SR',
|
||||
},
|
||||
})
|
||||
const hasHealthCheckSr = this._healthCheckSr !== undefined
|
||||
this.transfer = task.wrapFn(this.transfer)
|
||||
this.cleanup = task.wrapFn(this.cleanup, !hasHealthCheckSr)
|
||||
this.healthCheck = task.wrapFn(this.healthCheck, hasHealthCheckSr)
|
||||
this._prepare = task.wrapInside(this._prepare)
|
||||
this.transfer = task.wrapInside(this.transfer)
|
||||
if (this._healthCheckSr !== undefined) {
|
||||
this.cleanup = task.wrapInside(this.cleanup)
|
||||
this.healthCheck = task.wrap(this.healthCheck)
|
||||
} else {
|
||||
this.cleanup = task.wrap(this.cleanup)
|
||||
}
|
||||
|
||||
return task.run(() => this._prepare(isFull))
|
||||
return this._prepare(isFull)
|
||||
}
|
||||
|
||||
async _prepare(isFull) {
|
||||
const settings = this._settings
|
||||
const { uuid: srUuid, $xapi: xapi } = this._sr
|
||||
const vmUuid = this._vmUuid
|
||||
const scheduleId = this._scheduleId
|
||||
const scheduleId = this._schedule.id
|
||||
|
||||
// delete previous interrupted copies
|
||||
ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vmUuid), vm => vm.$destroy))
|
||||
|
||||
this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
|
||||
const allEntries = listReplicatedVms(xapi, scheduleId, srUuid, vmUuid)
|
||||
|
||||
if (settings.deleteFirst) {
|
||||
// In the snapshot-based flow a non-snapshot VM (the live target) coexists with its
|
||||
// snapshots (one per transfer). That VM must not be subject to retention — only its
|
||||
// snapshots are. Build the set of VM refs that already have snapshots in the list so
|
||||
// we can exclude them, while keeping old-style non-snapshot VMs (no snapshots).
|
||||
const vmRefsWithSnapshots = new Set(allEntries.filter(e => e.is_a_snapshot).map(e => e.snapshot_of))
|
||||
const retentionEntries = allEntries.filter(e => e.is_a_snapshot || !vmRefsWithSnapshots.has(e.$ref))
|
||||
retentionEntries.sort(compareReplicatedVmDatetime)
|
||||
this._oldEntries = getOldEntries(settings.copyRetention - 1, retentionEntries)
|
||||
|
||||
if (settings.deleteFirst && settings.skipDeleteOldEntries) {
|
||||
// we want to keep the baseVM when copying a delta
|
||||
// even if we want to keep only one after
|
||||
let mostRecentEntry
|
||||
@@ -86,45 +234,53 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
await this._deleteOldEntries()
|
||||
if (!this._settings.skipDeleteOldEntries) {
|
||||
await this._deleteOldEntries()
|
||||
}
|
||||
}
|
||||
|
||||
async _deleteOldEntries() {
|
||||
return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
|
||||
return asyncMapSettled(this._oldEntries, vm => vm.$destroy({ bypassBlockedOperation: true }))
|
||||
}
|
||||
|
||||
#decorateVmMetadata(backup) {
|
||||
#decorateVmMetadata(backup, timestamp) {
|
||||
const { _warmMigration } = this._settings
|
||||
const sr = this._sr
|
||||
const vm = backup.vm
|
||||
const job = this._job
|
||||
const scheduleId = this._schedule.id
|
||||
|
||||
// update other_config data as soon as possible to ensure the next job
|
||||
// will be able to detect any partial transfer and lean them
|
||||
vm.other_config[COPY_OF] = vm.uuid
|
||||
vm.other_config[JOB_ID] = job.id
|
||||
vm.other_config[SCHEDULE_ID] = scheduleId
|
||||
vm.other_config[REPLICATED_TO_SR_UUID] = sr.uuid
|
||||
// set the timestamp in the past to ensure any incomplete VM will be deleted on next run
|
||||
vm.other_config[DATETIME] = formatFilenameDate(0)
|
||||
|
||||
vm.blocked_operations = {
|
||||
start: 'Start operation for this vm is blocked, clone it if you want to use it.',
|
||||
start_on: 'Start operation for this vm is blocked, clone it if you want to use it.',
|
||||
}
|
||||
|
||||
if (!_warmMigration) {
|
||||
vm.tags.push('Continuous Replication')
|
||||
}
|
||||
// extracting the uuid of each delta vdi on the source
|
||||
// get all in one pass, since there is a lot of objects
|
||||
const sourceVdiUuids = Object.values(backup.vdis)
|
||||
.map(({ other_config }) => other_config[BASE_DELTA_VDI])
|
||||
// full vdi don't have a base
|
||||
.filter(_ => !!_)
|
||||
// @todo use index ?
|
||||
|
||||
const replicatedVdis = sr.$VDIs.filter(vdi => {
|
||||
// REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
|
||||
return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
|
||||
})
|
||||
|
||||
Object.values(backup.vdis).forEach(vdi => {
|
||||
vdi.other_config[COPY_OF] = vdi.uuid
|
||||
if (sourceVdiUuids.length > 0) {
|
||||
const baseReplicatedTo = replicatedVdis.find(
|
||||
replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
|
||||
)
|
||||
// baseReplicatedTo can be undefined if a new disk is added and other are already replicated
|
||||
vdi.baseVdi = baseReplicatedTo
|
||||
vdi.other_config[JOB_ID] = job.id
|
||||
vdi.other_config[SCHEDULE_ID] = scheduleId
|
||||
vdi.other_config[REPLICATED_TO_SR_UUID] = sr.uuid
|
||||
vdi.other_config[VM_UUID] = vm.uuid
|
||||
|
||||
const baseDeltaVdiUuid = vdi.other_config[BASE_DELTA_VDI]
|
||||
if (baseDeltaVdiUuid !== undefined) {
|
||||
// reuse the validated mapping built by checkBaseVdis
|
||||
vdi.baseVdi = this.#baseVdisBySourceUuid.get(baseDeltaVdiUuid)
|
||||
} else {
|
||||
// first replication of this disk
|
||||
// first replication of this disk (full, no base)
|
||||
vdi.baseVdi = undefined
|
||||
}
|
||||
// ensure the VDI are created on the target SR
|
||||
@@ -138,19 +294,43 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
const { _warmMigration } = this._settings
|
||||
const sr = this._sr
|
||||
const job = this._job
|
||||
const scheduleId = this._scheduleId
|
||||
|
||||
const schedule = this._schedule
|
||||
const scheduleId = schedule.id
|
||||
const { uuid: srUuid, $xapi: xapi } = sr
|
||||
|
||||
let targetVmRef
|
||||
await Task.run({ name: 'transfer' }, async () => {
|
||||
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
|
||||
// size is mandatory to ensure the task have the right data
|
||||
await Task.run({ properties: { name: 'transfer' } }, async () => {
|
||||
targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport, timestamp), sr, {
|
||||
targetRef: this._targetVmRef,
|
||||
})
|
||||
// this also ensure the data are up to date on the snapshot
|
||||
await setVmOtherConfig(xapi, targetVmRef, {
|
||||
timestamp, // updated at the end to mark the transfer as complete
|
||||
jobId: job.id,
|
||||
scheduleId,
|
||||
vmUuid: vm.uuid,
|
||||
srUuid,
|
||||
})
|
||||
const size = Object.values(deltaExport.disks).reduce(
|
||||
(sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
|
||||
0
|
||||
)
|
||||
await xapi.setField(
|
||||
'VM',
|
||||
targetVmRef,
|
||||
'name_description',
|
||||
deltaExport.vm.name_description +
|
||||
` -- last replication: ${formatFilenameDate(timestamp)} ${humanFormat.bytes(size)} read`
|
||||
)
|
||||
// take a snapshot to ensure these data are not modified until next snapshot
|
||||
await Task.run({ properties: { name: 'target snapshot' } }, async () => {
|
||||
await xapi.VM_snapshot(targetVmRef, {
|
||||
name_label: `${vm.name_label} - ${job.name} / ${schedule.name} ${formatFilenameDate(timestamp)}`,
|
||||
})
|
||||
})
|
||||
|
||||
return {
|
||||
size: Object.values(deltaExport.disks).reduce(
|
||||
(sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
|
||||
0
|
||||
),
|
||||
size,
|
||||
}
|
||||
})
|
||||
this._targetVmRef = targetVmRef
|
||||
@@ -161,20 +341,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
|
||||
!_warmMigration &&
|
||||
targetVm.ha_restart_priority !== '' &&
|
||||
Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
|
||||
targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
|
||||
asyncMap(['start', 'start_on'], op =>
|
||||
targetVm.update_blocked_operations(
|
||||
op,
|
||||
'Start operation for this vm is blocked, clone it if you want to use it.'
|
||||
)
|
||||
),
|
||||
setVmOtherConfig(xapi, targetVmRef, {
|
||||
timestamp,
|
||||
jobId: job.id,
|
||||
scheduleId,
|
||||
vmUuid: vm.uuid,
|
||||
srUuid,
|
||||
}),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { compareTimestamp } from '../../RemoteAdapter.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AbstractAggregatedRemoteWriter')
|
||||
|
||||
export const DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY = 2
|
||||
export class AbstractAggregatedRemoteWriter {
|
||||
#BackupWriter
|
||||
#adapters
|
||||
get adapters() {
|
||||
return this.#adapters
|
||||
}
|
||||
|
||||
mainWriter
|
||||
#props
|
||||
get props() {
|
||||
return this.#props
|
||||
}
|
||||
#oldBackups
|
||||
|
||||
#writers = []
|
||||
get writers() {
|
||||
return this.#writers
|
||||
}
|
||||
constructor({ adapters, BackupWriter, ...props }) {
|
||||
debug('instantiate AggregatedFullRemoteWriter')
|
||||
if (Object.keys(adapters).length < 1) {
|
||||
throw new Error('AggregatedRemoteWriter need at least one backup repository to work ')
|
||||
}
|
||||
this.#props = props
|
||||
this.#adapters = adapters
|
||||
this.#BackupWriter = BackupWriter
|
||||
}
|
||||
|
||||
getWriterByAdapter(adapter) {
|
||||
for (const writer of this.#writers) {
|
||||
if (writer._adapter === adapter) {
|
||||
return writer
|
||||
}
|
||||
}
|
||||
throw new Error("Can't find writer for adapter")
|
||||
}
|
||||
|
||||
/**
|
||||
* select the best adapter candidate to write this backup
|
||||
* if all the adapter infos expose an available info use the one with the more free space
|
||||
* if none of the adapter exposes this info : choose at random
|
||||
* if mixed : throw an error
|
||||
* @returns {RemoteAdapter}
|
||||
*/
|
||||
async computeAdapterCandidate() {
|
||||
const adapters = Object.values(this.#adapters)
|
||||
const infos = await Promise.all(
|
||||
adapters.map(async adapter => {
|
||||
const infos = await adapter._handler.getInfo()
|
||||
return { adapter, infos }
|
||||
})
|
||||
)
|
||||
const withLimit = infos.filter(({ infos: { available } }) => available > 0).length
|
||||
if (withLimit !== 0 && withLimit !== infos.length) {
|
||||
throw new Error(`an Aggregate remote can't use a mix of limited and unlimited storages`)
|
||||
}
|
||||
let candidates
|
||||
if (withLimit === 0) {
|
||||
debug('no storage limit , choose at random')
|
||||
candidates = Object.values(this.adapters)
|
||||
} else {
|
||||
debug('storage limit , choose at random between those with the max space avilable')
|
||||
let maxSpace = 0
|
||||
infos.forEach(({ infos: { available } }) => {
|
||||
if (available > maxSpace) {
|
||||
maxSpace = available
|
||||
}
|
||||
})
|
||||
|
||||
// candidates are the adapter with the same free space , equal to the max free space
|
||||
candidates = infos.filter(({ infos }) => infos.available === maxSpace).map(({ adapter }) => adapter)
|
||||
}
|
||||
const adapterCandidate = candidates[Math.floor(Math.random() * candidates.length)]
|
||||
debug('adapter candidate is selected ', adapterCandidate._handler._remote.id)
|
||||
return adapterCandidate
|
||||
}
|
||||
|
||||
async setupWriters() {
|
||||
debug('Setup writers ')
|
||||
const settings = this.#props.settings
|
||||
const allSettings = this.#props.job.settings
|
||||
|
||||
Object.entries(this.#adapters).forEach(([remoteId, adapter]) => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[remoteId],
|
||||
skipDeleteOldEntries: true, // delete is handled globally
|
||||
}
|
||||
if (targetSettings.exportRetention !== 0) {
|
||||
const writer = new this.#BackupWriter({
|
||||
...this.#props,
|
||||
adapter,
|
||||
remoteId,
|
||||
settings: targetSettings,
|
||||
})
|
||||
this.#writers.push(writer)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async getEntriesPerAdapter(adapter) {
|
||||
const scheduleId = this.#props.schedule.id
|
||||
const vmUuid = this.#props.vmUuid
|
||||
return (await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)).map(entry => ({
|
||||
...entry,
|
||||
adapter,
|
||||
}))
|
||||
}
|
||||
|
||||
async setOldBackups() {
|
||||
const entriesPerAdapter = await Promise.all(
|
||||
Object.values(this.#adapters).map(adapter => this.getEntriesPerAdapter(adapter))
|
||||
)
|
||||
const entries = entriesPerAdapter
|
||||
.flat(1)
|
||||
.filter(_ => !!_)
|
||||
.sort(compareTimestamp)
|
||||
|
||||
const settings = this.#props.settings
|
||||
this.#oldBackups = getOldEntries(settings.exportRetention - 1, entries, {
|
||||
longTermRetention: settings.longTermRetention,
|
||||
timezone: settings.timezone,
|
||||
})
|
||||
debug(`got ${this.#oldBackups.length} old backup(s) to delete `)
|
||||
}
|
||||
|
||||
async deleteOldBackupsOnAdapter(adapter, backup) {
|
||||
throw new Error('not implemented')
|
||||
}
|
||||
|
||||
async deleteOldBackups() {
|
||||
const byAdapters = new Map()
|
||||
this.#oldBackups.forEach(({ adapter, ...backup }) => {
|
||||
const current = byAdapters.has(adapter) ? byAdapters.get(adapter) : []
|
||||
current.push(backup)
|
||||
byAdapters.set(adapter, current)
|
||||
})
|
||||
await asyncEach(
|
||||
byAdapters.entries(),
|
||||
async ([adapter, backups]) => {
|
||||
debug(`delete ${backups.length} old backup(s) to delete from ${adapter._handler._remote.id}`)
|
||||
await this.deleteOldBackupsOnAdapter(adapter, backups)
|
||||
},
|
||||
{
|
||||
concurrency: this.#props.deleteConcurrency ?? DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { getOldEntries } from '../../_getOldEntries.mjs'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { compareReplicatedVmDatetime, listReplicatedVms } from './_listReplicatedVms.mjs'
|
||||
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
||||
|
||||
const { debug } = createLogger('xo:backups:AbstractAggregatedXapiWriter')
|
||||
export class AbstractAggregatedXapiWriter {
|
||||
#ReplicationWriter
|
||||
#storageRepositories
|
||||
|
||||
#writers = []
|
||||
get writers() {
|
||||
return this.#writers
|
||||
}
|
||||
mainWriter
|
||||
#props
|
||||
get props() {
|
||||
return this.#props
|
||||
}
|
||||
#oldVmReplicaList = []
|
||||
|
||||
constructor({ srs, ReplicationWriter, ...props }) {
|
||||
debug('instantiate AbstractAggregatedXapiWriter')
|
||||
if (srs.length < 1) {
|
||||
throw new Error('AggregatedXapiWriter need at least one storage repository to work ')
|
||||
}
|
||||
this.#props = props
|
||||
this.#ReplicationWriter = ReplicationWriter
|
||||
this.#storageRepositories = srs
|
||||
}
|
||||
|
||||
getWriterBySr(storageRepository) {
|
||||
for (const writer of this.#writers) {
|
||||
if (writer._sr === storageRepository) {
|
||||
return writer
|
||||
}
|
||||
}
|
||||
throw new Error("Can't find writer for storage repository")
|
||||
}
|
||||
|
||||
async computeSRCandidate() {
|
||||
let maxSpace = 0
|
||||
this.#storageRepositories.forEach(({ physical_utilisation, physical_size }) => {
|
||||
if (physical_size - physical_utilisation > maxSpace) {
|
||||
maxSpace = physical_size - physical_utilisation
|
||||
}
|
||||
})
|
||||
debug(`computeSRCandidate found the max free space ${maxSpace} for vm${this.#props.vmUuid}`)
|
||||
|
||||
const srCandidates = this.#storageRepositories.filter(
|
||||
({ physical_utilisation, physical_size }) => maxSpace === physical_size - physical_utilisation
|
||||
)
|
||||
// one at random from the sr with the most free space
|
||||
const candidate = srCandidates[Math.floor(Math.random() * srCandidates.length)]
|
||||
|
||||
debug(
|
||||
`computeSRCandidate found ${srCandidates.length} sr with ${maxSpace} free size, and chose ${candidate.name_label} for vm${this.#props.vmUuid}`
|
||||
)
|
||||
return candidate
|
||||
}
|
||||
|
||||
setOldReplicaList() {
|
||||
debug(`setOldReplicaList for vm ${this.#props.vmUuid}`)
|
||||
const scheduleId = this.#props.scheduleId
|
||||
const vmUuid = this.#props.vmUuid
|
||||
const settings = this.#props.settings
|
||||
|
||||
const replicatedVms = this.#storageRepositories
|
||||
.map(sr => {
|
||||
return listReplicatedVms(sr.$xapi, scheduleId, sr.uuid, vmUuid)
|
||||
})
|
||||
.flat(1)
|
||||
.filter(_ => !!_)
|
||||
.sort(compareReplicatedVmDatetime)
|
||||
|
||||
this.#oldVmReplicaList = getOldEntries(settings.copyRetention - 1, replicatedVms)
|
||||
|
||||
debug(
|
||||
`setOldReplicaList found ${replicatedVms.length} replica, and will delete ${this.#oldVmReplicaList.length} (retention ${settings.copyRetention}) for vm ${this.#props.vmUuid}`
|
||||
)
|
||||
}
|
||||
|
||||
async deleteOldReplicas() {
|
||||
debug(`deleteOldReplicas will delete ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
// destroy the VM on the right xapi
|
||||
await asyncMapSettled(this.#oldVmReplicaList, async vm => {
|
||||
debug('will delete old replica', vm.name_label)
|
||||
await vm.$xapi.VM_destroy(vm.$ref)
|
||||
})
|
||||
debug(`deleteOldReplicas deleted ${this.#oldVmReplicaList.length} replicated vms for vm ${this.#props.vmUuid}`)
|
||||
}
|
||||
|
||||
async setupWriters() {
|
||||
debug('Setup writers ')
|
||||
const settings = this.#props.settings
|
||||
const allSettings = this.#props.job.settings
|
||||
|
||||
this.#storageRepositories.forEach(storageRepository => {
|
||||
const targetSettings = {
|
||||
...settings,
|
||||
...allSettings[storageRepository.uuid],
|
||||
skipDeleteOldEntries: true, // delete is handled globally
|
||||
}
|
||||
if (targetSettings.copyRetention !== 0) {
|
||||
const writer = new this.#ReplicationWriter({
|
||||
...this.#props,
|
||||
sr: storageRepository,
|
||||
settings: targetSettings,
|
||||
})
|
||||
this.#writers.push(writer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,9 @@
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
|
||||
export class AbstractWriter {
|
||||
constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
|
||||
constructor({ config, healthCheckSr, job, vmUuid, schedule, settings }) {
|
||||
this._config = config
|
||||
this._healthCheckSr = healthCheckSr
|
||||
this._job = job
|
||||
this._scheduleId = scheduleId
|
||||
this._schedule = schedule
|
||||
this._settings = settings
|
||||
this._vmUuid = vmUuid
|
||||
}
|
||||
@@ -16,14 +13,4 @@ export class AbstractWriter {
|
||||
afterBackup() {}
|
||||
|
||||
healthCheck(sr) {}
|
||||
|
||||
_isAlreadyTransferred(timestamp) {
|
||||
const vmUuid = this._vmUuid
|
||||
const adapter = this._adapter
|
||||
const backupDir = getVmBackupDir(vmUuid)
|
||||
try {
|
||||
const actualMetadata = JSON.parse(adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`))
|
||||
return actualMetadata
|
||||
} catch (error) {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { join } from 'node:path'
|
||||
import { Task } from '@vates/task'
|
||||
import assert from 'node:assert'
|
||||
|
||||
import { formatFilenameDate } from '../../_filenameDate.mjs'
|
||||
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
||||
import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
|
||||
import { ImportVmBackup } from '../../ImportVmBackup.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import * as MergeWorker from '../../merge-worker/index.mjs'
|
||||
import ms from 'ms'
|
||||
import { getEntryStatus } from '../../_getOldEntries.mjs'
|
||||
|
||||
const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
|
||||
|
||||
@@ -27,7 +28,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
|
||||
async _cleanVm(options) {
|
||||
try {
|
||||
return await Task.run({ name: 'clean-vm' }, () => {
|
||||
return await Task.run({ properties: { name: 'clean-vm' } }, () => {
|
||||
return this._adapter.cleanVm(this._vmBackupDir, {
|
||||
...options,
|
||||
fixMetadata: true,
|
||||
@@ -47,6 +48,19 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
}
|
||||
}
|
||||
|
||||
async getLongTermRetentionTags(currentEntry) {
|
||||
const settings = this._settings
|
||||
const scheduleId = this._schedule.id
|
||||
const vmUuid = this._vmUuid
|
||||
const adapter = this._adapter
|
||||
|
||||
const entries = await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)
|
||||
entries.push(currentEntry)
|
||||
const buckets = getEntryStatus(entries, currentEntry, settings.longTermRetention, settings.timezone)
|
||||
|
||||
return buckets.map(({ duration, dateBucket }) => `${duration}=${dateBucket}`)
|
||||
}
|
||||
|
||||
async beforeBackup() {
|
||||
const { handler } = this._adapter
|
||||
const vmBackupDir = this._vmBackupDir
|
||||
@@ -86,7 +100,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
}
|
||||
return Task.run(
|
||||
{
|
||||
name: 'health check',
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
async () => {
|
||||
const xapi = sr.$xapi
|
||||
@@ -99,7 +113,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
srUuid,
|
||||
xapi,
|
||||
settings: {
|
||||
additionnalVmTag: 'xo:no-bak=Health Check',
|
||||
additionalVmTag: 'xo:no-bak=Health Check',
|
||||
},
|
||||
}).run()
|
||||
let restoredVm
|
||||
@@ -116,7 +130,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
|
||||
xapi,
|
||||
}).run()
|
||||
} finally {
|
||||
await xapi.VM_destroy(restoredVm.$ref)
|
||||
await xapi.VM_destroy(restoredVm.$ref, { bypassBlockedOperation: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { Task } from '@vates/task'
|
||||
|
||||
import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
|
||||
import { Task } from '../../Task.mjs'
|
||||
import ms from 'ms'
|
||||
|
||||
export const MixinXapiWriter = (BaseClass = Object) =>
|
||||
@@ -29,11 +29,11 @@ export const MixinXapiWriter = (BaseClass = Object) =>
|
||||
// the SR that the VM has been replicated on
|
||||
const sr = this._sr
|
||||
assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
|
||||
assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
|
||||
assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transferred to be health checked')
|
||||
// copy VM
|
||||
return Task.run(
|
||||
{
|
||||
name: 'health check',
|
||||
properties: { name: 'health check' },
|
||||
},
|
||||
async () => {
|
||||
const { $xapi: xapi } = sr
|
||||
@@ -47,12 +47,12 @@ export const MixinXapiWriter = (BaseClass = Object) =>
|
||||
}
|
||||
if (await this.#isAlreadyOnHealthCheckSr(baseVm)) {
|
||||
healthCheckVmRef = await Task.run(
|
||||
{ name: 'cloning-vm' },
|
||||
{ properties: { name: 'cloning-vm' } },
|
||||
async () => await xapi.callAsync('VM.clone', this._targetVmRef, `Health Check - ${baseVm.name_label}`)
|
||||
)
|
||||
} else {
|
||||
healthCheckVmRef = await Task.run(
|
||||
{ name: 'copying-vm' },
|
||||
{ properties: { name: 'copying-vm' } },
|
||||
async () =>
|
||||
await xapi.callAsync(
|
||||
'VM.copy',
|
||||
@@ -72,7 +72,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
|
||||
xapi,
|
||||
}).run()
|
||||
} finally {
|
||||
healthCheckVmRef && (await xapi.VM_destroy(healthCheckVmRef))
|
||||
healthCheckVmRef && (await xapi.VM_destroy(healthCheckVmRef, { bypassBlockedOperation: true }))
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -5,24 +5,31 @@ const getReplicatedVmDatetime = vm => {
|
||||
return datetime
|
||||
}
|
||||
|
||||
const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
|
||||
export const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Xapi} xapi
|
||||
* @param {string} scheduleOrJobId
|
||||
* @param {string} srUuid
|
||||
* @param {string} vmUuid
|
||||
* @returns {Array<XoVm>}
|
||||
*/
|
||||
export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
|
||||
const { all } = xapi.objects
|
||||
const all = xapi.objects.indexes.type.VM
|
||||
const vms = {}
|
||||
for (const key in all) {
|
||||
const object = all[key]
|
||||
const oc = object.other_config
|
||||
if (
|
||||
object.$type === 'VM' &&
|
||||
!object.is_a_snapshot &&
|
||||
!object.is_a_template &&
|
||||
'start' in object.blocked_operations &&
|
||||
(oc[JOB_ID] === scheduleOrJobId || oc[SCHEDULE_ID] === scheduleOrJobId) &&
|
||||
oc[REPLICATED_TO_SR_UUID] === srUuid &&
|
||||
(oc[VM_UUID] === vmUuid ||
|
||||
// 2018-03-28, JFT: to catch VMs replicated before this fix
|
||||
oc[VM_UUID] === undefined)
|
||||
oc[VM_UUID] === vmUuid &&
|
||||
// Old-style replication: one VM per transfer (non-snapshot, start blocked)
|
||||
// New-style replication: snapshots of the target VM represent each transfer
|
||||
(!object.is_a_snapshot ? 'start' in object.blocked_operations : true)
|
||||
) {
|
||||
vms[object.$id] = object
|
||||
}
|
||||
|
||||
325
@xen-orchestra/backups/disks/MergeRemoteDisk.mjs
Normal file
325
@xen-orchestra/backups/disks/MergeRemoteDisk.mjs
Normal file
@@ -0,0 +1,325 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import assert from 'assert'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
import { basename, dirname } from 'path'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { relativeFromFile } from '@xen-orchestra/fs/path'
|
||||
|
||||
// @ts-ignore
|
||||
const { warn } = createLogger('remote-disk:merge')
|
||||
|
||||
/**
|
||||
* @typedef {Object} MergeState
|
||||
* @property {{ uuid: string }} child
|
||||
* @property {{ uuid: string }} parent
|
||||
* @property { string[] | undefined} chain
|
||||
* @property {number} currentBlock
|
||||
* @property {number} mergedDataSize
|
||||
* @property {'mergeBlocks' | 'cleanup'} step
|
||||
* @property {number} diskSize
|
||||
* @typedef {(message: string, data?: Record<string, unknown>) => void} Logger
|
||||
*/
|
||||
|
||||
export class MergeRemoteDisk {
|
||||
/**
|
||||
* @type {MergeState}
|
||||
*/
|
||||
#state = {
|
||||
child: { uuid: '0' },
|
||||
parent: { uuid: '0' },
|
||||
chain: undefined,
|
||||
currentBlock: 0,
|
||||
mergedDataSize: 0,
|
||||
step: 'mergeBlocks',
|
||||
diskSize: 0,
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#statePath = ''
|
||||
|
||||
/**
|
||||
* @type {boolean}
|
||||
*/
|
||||
#isResuming
|
||||
|
||||
/**
|
||||
* @type {Logger | Function}
|
||||
*/
|
||||
#logInfo
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#mergeBlockConcurrency
|
||||
|
||||
/**
|
||||
* @type {Function}
|
||||
*/
|
||||
#onProgress
|
||||
|
||||
/**
|
||||
* @type {boolean}
|
||||
*/
|
||||
#removeUnused
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#writeStateDelay
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#lastStateWrittenAt
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @param {FileAccessor} handler
|
||||
* @param {Object} params
|
||||
* @param {Function} [params.onProgress]
|
||||
* @param {Logger | Function} [params.logInfo]
|
||||
* @param {boolean} [params.removeUnused]
|
||||
* @param {number} [params.mergeBlockConcurrency]
|
||||
* @param {number} [params.writeStateDelay]
|
||||
*/
|
||||
constructor(
|
||||
handler,
|
||||
{
|
||||
onProgress = () => {},
|
||||
logInfo = () => {},
|
||||
removeUnused = false,
|
||||
mergeBlockConcurrency = 2,
|
||||
writeStateDelay = 10e3,
|
||||
}
|
||||
) {
|
||||
this.#handler = handler
|
||||
this.#logInfo = logInfo
|
||||
this.#onProgress = onProgress
|
||||
this.#removeUnused = removeUnused
|
||||
this.#mergeBlockConcurrency = mergeBlockConcurrency
|
||||
this.#writeStateDelay = writeStateDelay
|
||||
|
||||
this.#isResuming = false
|
||||
this.#lastStateWrittenAt = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @returns {Promise<boolean>} isResuming
|
||||
*/
|
||||
async isResuming(parentDisk) {
|
||||
try {
|
||||
await this.#handler.readFile(
|
||||
dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async merge(parentDisk, childDisk) {
|
||||
this.#statePath = dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
|
||||
|
||||
try {
|
||||
const mergeStateContent = await this.#handler.readFile(this.#statePath)
|
||||
this.#state = JSON.parse(mergeStateContent)
|
||||
|
||||
// work-around a bug introduce in 97d94b795
|
||||
//
|
||||
// currentBlock could be `null` due to the JSON.stringify of a `NaN` value
|
||||
if (this.#state?.currentBlock === null) this.#state.currentBlock = 0
|
||||
this.#isResuming = true
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code !== 'ENOENT') {
|
||||
warn('problem while checking the merge state', { error })
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
/* eslint-disable no-fallthrough */
|
||||
switch (this.#state?.step ?? 'mergeBlocks') {
|
||||
case 'mergeBlocks':
|
||||
await this.#step_mergeBlocks(parentDisk, childDisk)
|
||||
case 'cleanup':
|
||||
await this.#step_cleanup(parentDisk, childDisk)
|
||||
return this.#cleanup(parentDisk, childDisk, true)
|
||||
default:
|
||||
warn(`Step ${this.#state?.step} is unknown`, { state: this.#state })
|
||||
}
|
||||
/* eslint-enable no-fallthrough */
|
||||
} catch (error) {
|
||||
await this.#cleanup(parentDisk, childDisk, false)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async #writeState() {
|
||||
try {
|
||||
await this.#handler.writeFile(this.#statePath, JSON.stringify(this.#state), { flags: 'w' })
|
||||
} catch (err) {
|
||||
warn('failed to write merge state', { error: err })
|
||||
}
|
||||
}
|
||||
|
||||
async #writeStateThrottled() {
|
||||
const now = Date.now()
|
||||
if (now - this.#lastStateWrittenAt > this.#writeStateDelay) {
|
||||
this.#lastStateWrittenAt = now
|
||||
await this.#writeState()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #step_mergeBlocks(parentDisk, childDisk) {
|
||||
const getMaxBlockCount = childDisk.getMaxBlockCount()
|
||||
await parentDisk.resize(getMaxBlockCount)
|
||||
|
||||
if (this.#isResuming) {
|
||||
const alreadyMergedBlocks = []
|
||||
for (let blockId = 0; blockId < this.#state.currentBlock; blockId++) {
|
||||
if (childDisk.hasBlock(blockId)) {
|
||||
alreadyMergedBlocks.push(blockId)
|
||||
}
|
||||
}
|
||||
|
||||
parentDisk.setAllocatedBlocks(alreadyMergedBlocks)
|
||||
} else {
|
||||
this.#state.child = { uuid: childDisk.getUuid() ?? undefined }
|
||||
this.#state.parent = { uuid: parentDisk.getUuid() ?? undefined }
|
||||
this.#state.chain = [parentDisk.getPath(), ...childDisk.getPaths()].map(path =>
|
||||
relativeFromFile(this.#statePath, path)
|
||||
)
|
||||
|
||||
// Finds first allocated block for the 2 following loops
|
||||
while (this.#state.currentBlock < getMaxBlockCount && !childDisk.hasBlock(this.#state.currentBlock)) {
|
||||
++this.#state.currentBlock
|
||||
}
|
||||
await this.#writeState()
|
||||
}
|
||||
|
||||
await this.#mergeBlocks(parentDisk, childDisk)
|
||||
await parentDisk.flushMetadata(childDisk)
|
||||
await parentDisk.mergeMetadata(childDisk)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #mergeBlocks(parentDisk, childDisk) {
|
||||
this.#mergeBlockConcurrency =
|
||||
(await parentDisk.canMergeConcurently()) && (await childDisk.canMergeConcurently())
|
||||
? this.#mergeBlockConcurrency
|
||||
: 1
|
||||
|
||||
const toMerge = []
|
||||
for (let block = this.#state.currentBlock; block < childDisk.getMaxBlockCount(); block++) {
|
||||
if (childDisk.hasBlock(block)) {
|
||||
toMerge.push(block)
|
||||
}
|
||||
}
|
||||
|
||||
const nBlocks = toMerge.length
|
||||
this.#onProgress({ total: nBlocks, done: 0 })
|
||||
|
||||
const merging = new Set()
|
||||
let counter = 0
|
||||
await asyncEach(
|
||||
toMerge,
|
||||
async blockId => {
|
||||
merging.add(blockId)
|
||||
|
||||
const blockSize = await parentDisk.mergeBlock(childDisk, blockId, this.#isResuming)
|
||||
this.#state.mergedDataSize += blockSize
|
||||
|
||||
this.#state.currentBlock = Math.min(...merging) - 1
|
||||
merging.delete(blockId)
|
||||
|
||||
this.#onProgress({ total: nBlocks, done: counter + 1 })
|
||||
counter++
|
||||
await this.#writeStateThrottled()
|
||||
},
|
||||
{ concurrency: this.#mergeBlockConcurrency }
|
||||
)
|
||||
|
||||
await this.#writeState()
|
||||
|
||||
this.#state.diskSize = childDisk.getSizeOnDisk()
|
||||
|
||||
this.#onProgress({ total: nBlocks, done: nBlocks })
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
*/
|
||||
async #step_cleanup(parentDisk, childDisk) {
|
||||
assert.notEqual(this.#state, undefined)
|
||||
this.#state.step = 'cleanup'
|
||||
await this.#writeState()
|
||||
|
||||
const mergeTargetPath = childDisk.getPath()
|
||||
|
||||
// delete intermediate children if needed
|
||||
if (this.#removeUnused) {
|
||||
await childDisk.unlink()
|
||||
}
|
||||
|
||||
try {
|
||||
await parentDisk.rename(mergeTargetPath)
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code === 'ENOENT' && this.#isResuming) {
|
||||
// @ts-ignore
|
||||
this.#logInfo(`the parent disk was already renamed`, {
|
||||
parent: parentDisk.getPath(),
|
||||
mergeTarget: mergeTargetPath,
|
||||
})
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteDisk} parentDisk
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {boolean} cleanStateFile
|
||||
*
|
||||
* @returns {Promise<{mergedDataSize: number, finalDiskSize: number}>} result
|
||||
*/
|
||||
async #cleanup(parentDisk, childDisk, cleanStateFile) {
|
||||
const finalDiskSize = this.#state?.diskSize ?? 0
|
||||
const mergedDataSize = this.#state?.mergedDataSize ?? 0
|
||||
await parentDisk.close().catch(warn)
|
||||
await childDisk.close().catch(warn)
|
||||
|
||||
if (cleanStateFile) {
|
||||
await this.#handler.unlink(this.#statePath).catch(warn)
|
||||
}
|
||||
|
||||
return { mergedDataSize, finalDiskSize }
|
||||
}
|
||||
}
|
||||
223
@xen-orchestra/backups/disks/RemoteDisk.mjs
Normal file
223
@xen-orchestra/backups/disks/RemoteDisk.mjs
Normal file
@@ -0,0 +1,223 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
*/
|
||||
|
||||
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
export class RemoteDisk extends RandomAccessDisk {
|
||||
/**
|
||||
* Abstract
|
||||
* @param {Object} options
|
||||
* @param {boolean} options.force
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options = { force: false }) {
|
||||
throw new Error(`init must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
throw new Error(`close must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
throw new Error(`getVirtualSize must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
throw new Error(`getSizeOnDisk must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
throw new Error(`getBlockSize must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
throw new Error(`getPath must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Returns an array of disk paths.
|
||||
*
|
||||
* @returns {string[]}
|
||||
*/
|
||||
getPaths() {
|
||||
throw new Error(`getPaths must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
throw new Error(`getUuid must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
throw new Error(`canMergeConcurently must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} getMaxBlockCount
|
||||
*/
|
||||
getMaxBlockCount() {
|
||||
return Math.ceil(this.getVirtualSize() / this.getBlockSize())
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
throw new Error(`hasBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
throw new Error(`getBlockIndexes must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
throw new Error(`instantiateParent must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Writes a full block.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
throw new Error(`writeBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
throw new Error(`readBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
throw new Error(`mergeBlock must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Manually set the disk allocated blocks.
|
||||
* @param {Array<number>} blockIds
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async setAllocatedBlocks(blockIds) {
|
||||
throw new Error(`setAllocatedBlocks must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @param {number} blockCount
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async resize(blockCount) {
|
||||
throw new Error(`resize must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
throw new Error(`flushMetadata must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
mergeMetadata(childDisk) {
|
||||
throw new Error(`mergeMetadata must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
throw new Error(`isDifferencing must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
throw new Error(`rename must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Deletes alias/disk/disks
|
||||
*/
|
||||
async unlink() {
|
||||
throw new Error(`unlink must be implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
async openParent() {
|
||||
const parent = await super.openParent()
|
||||
if (!(parent instanceof RemoteDisk)) {
|
||||
throw new Error('parent of a RemoteDisk must be also a RemoteDisk')
|
||||
}
|
||||
return parent
|
||||
}
|
||||
}
|
||||
1119
@xen-orchestra/backups/disks/RemoteDisk.test.mjs
Normal file
1119
@xen-orchestra/backups/disks/RemoteDisk.test.mjs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,171 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').Disk} Disk
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
*/
|
||||
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { DISK_TYPES } from 'vhd-lib/_constants.js'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
|
||||
/**
|
||||
* Represents a remote VHD (Virtual Hard Disk) that extends RandomAccessDisk.
|
||||
*/
|
||||
export class RemoteVhd extends RandomAccessDisk {
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @type {VhdFile | VhdDirectory | undefined}
|
||||
*/
|
||||
#vhd
|
||||
|
||||
/**
|
||||
* @type {boolean | undefined}
|
||||
*/
|
||||
#isDifferencing
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
get path() {
|
||||
return this.#path
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getvirtualsize of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.footer.currentSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return 2 * 1024 * 1024
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init() {
|
||||
const { value, dispose } = await openVhd(this.#handler, this.#path)
|
||||
this.#vhd = value
|
||||
this.#dispose = dispose
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await this.#dispose()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call hasblock of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#vhd.containsBlock(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getBlockIndexes of a RemoteVhd before init`)
|
||||
}
|
||||
const index = []
|
||||
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
|
||||
if (this.hasBlock(blockIndex)) {
|
||||
index.push(blockIndex)
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>}
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhd before init`)
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @returns {RandomAccessDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call openParent of a RemoteVhd before init`)
|
||||
}
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
if (!parentPath) {
|
||||
throw new Error(`Disk ${this.#path} doesn't have parents`)
|
||||
}
|
||||
const parent = new RemoteVhd({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
if (this.#isDifferencing === undefined) {
|
||||
throw new Error(`can't call isDifferencing of a RemoteVhd before init`)
|
||||
}
|
||||
return this.#isDifferencing
|
||||
}
|
||||
}
|
||||
480
@xen-orchestra/backups/disks/RemoteVhdDisk.mjs
Normal file
480
@xen-orchestra/backups/disks/RemoteVhdDisk.mjs
Normal file
@@ -0,0 +1,480 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
|
||||
* @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
|
||||
* @typedef {import('vhd-lib/_createFooterHeader').VhdFooter} VhdFooter
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*
|
||||
|
||||
*/
|
||||
|
||||
import { openVhd, VhdAbstract, VhdDirectory } from 'vhd-lib'
|
||||
import { RemoteDisk } from './RemoteDisk.mjs'
|
||||
import { DISK_TYPES } from 'vhd-lib/_constants.js'
|
||||
import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
|
||||
import { stringify } from 'uuid'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RemoteVhdDiskChain } from './RemoteVhdDiskChain.mjs'
|
||||
|
||||
export class RemoteVhdDisk extends RemoteDisk {
|
||||
/**
|
||||
* @type {string}
|
||||
*/
|
||||
#path
|
||||
|
||||
/**
|
||||
* @type {FileAccessor}
|
||||
*/
|
||||
#handler
|
||||
|
||||
/**
|
||||
* @type {VhdFile | VhdDirectory | undefined}
|
||||
*/
|
||||
#vhd
|
||||
|
||||
/**
|
||||
* @type {boolean | undefined}
|
||||
*/
|
||||
#isDifferencing
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#blockSize = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#bitmapSize = 512
|
||||
|
||||
/**
|
||||
* @type {() => any}
|
||||
*/
|
||||
#dispose = () => {}
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
*/
|
||||
constructor({ handler, path }) {
|
||||
super()
|
||||
// @todo : ensure this is the full path from the root of the remote
|
||||
this.#path = path
|
||||
this.#handler = handler
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} [options]
|
||||
* @param {boolean} [options.force=false]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options = {}) {
|
||||
if (this.#vhd === undefined) {
|
||||
try {
|
||||
const { value, dispose } = await openVhd(this.#handler, await resolveVhdAlias(this.#handler, this.#path), {
|
||||
checkSecondFooter: !options.force,
|
||||
})
|
||||
this.#vhd = value
|
||||
|
||||
if ((await this.isDirectory()) && !isVhdAlias(this.#path)) {
|
||||
this.#vhd = undefined
|
||||
throw Object.assign(new Error("Can't init vhd directory without using alias"), { code: 'NOT_SUPPORTED' })
|
||||
}
|
||||
|
||||
this.#dispose = dispose
|
||||
await this.#vhd.readBlockAllocationTable()
|
||||
this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
|
||||
} catch (error) {
|
||||
await this.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* We replace the dispose function call so the disk can only be closed once.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
const dispose = this.#dispose
|
||||
this.#dispose = () => Promise.resolve()
|
||||
await dispose()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getVirtualSize of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#vhd.footer.currentSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getVirtualSize of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd.streamSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return this.#blockSize
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
return this.#path
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the disk path in an array.
|
||||
*
|
||||
* @returns {string[]}
|
||||
*/
|
||||
getPaths() {
|
||||
return [this.getPath()]
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getUid of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return stringify(this.#vhd.footer.uuid)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getParentUuid() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getParentUid of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return stringify(this.#vhd.header.parentUuid)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
return await this.isDirectory()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call hasblock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#vhd.containsBlock(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getBlockIndexes of a RemoteVhdDisk before init`)
|
||||
}
|
||||
const indexes = []
|
||||
for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
|
||||
if (this.hasBlock(blockIndex)) {
|
||||
indexes.push(blockIndex)
|
||||
}
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call instantiateParent of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
const parentPath = this.#vhd.header.parentUnicodeName
|
||||
const fullParentPath = join(dirname(this.#path), parentPath)
|
||||
|
||||
if (!parentPath) {
|
||||
throw new Error(`disk ${this.#path} doesn't have parents`)
|
||||
}
|
||||
|
||||
const parent = new RemoteVhdDisk({ handler: this.#handler, path: fullParentPath })
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a full block into this VHD.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
await this.#vhd.writeEntireBlock({
|
||||
id: diskBlock.index,
|
||||
buffer: Buffer.concat([Buffer.alloc(this.#bitmapSize, 255), diskBlock.data]),
|
||||
})
|
||||
|
||||
return this.getBlockSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
const { data } = await this.#vhd.readBlock(index)
|
||||
return {
|
||||
index,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
if (
|
||||
(childDisk instanceof RemoteVhdDisk || childDisk instanceof RemoteVhdDiskChain) &&
|
||||
(await this.isDirectory()) &&
|
||||
(await childDisk.isDirectory())
|
||||
) {
|
||||
try {
|
||||
await this.#handler.rename(childDisk.getBlockPath(index), this.getBlockPath(index))
|
||||
|
||||
this.setAllocatedBlocks([index])
|
||||
} catch (error) {
|
||||
// @ts-ignore
|
||||
if (error.code === 'ENOENT' && isResumingMerge === true) {
|
||||
// when resuming, the blocks moved since the last merge state write are
|
||||
// not in the child anymore but it should be ok
|
||||
|
||||
// it will throw an error if block is missing in parent
|
||||
// won't detect if the block was already in parent and is broken/missing in child
|
||||
|
||||
// since we can't know the initial size, this will create a discrepancy
|
||||
// on the size
|
||||
const { data } = await this.readBlock(index)
|
||||
if (data.length !== this.getBlockSize()) {
|
||||
throw error
|
||||
} else {
|
||||
this.setAllocatedBlocks([index])
|
||||
}
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return this.getBlockSize()
|
||||
} else {
|
||||
return this.writeBlock(await childDisk.readBlock(index))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a specific block path from the VHD directory.
|
||||
* @param {number} index
|
||||
* @returns {string} blockPath
|
||||
*/
|
||||
getBlockPath(index) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call readBlock of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
if (this.#vhd instanceof VhdDirectory) {
|
||||
return this.#vhd.getFullBlockPath(index)
|
||||
} else {
|
||||
throw new Error(`can't call getBlockPath of non directory VHD`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually set the disk allocated blocks.
|
||||
* @param {Array<number>} blockIds
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async setAllocatedBlocks(blockIds) {
|
||||
if (this.#vhd instanceof VhdDirectory) {
|
||||
this.#vhd.setAllocatedBlocks(blockIds)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure that the disk can handle at least the new block count.
|
||||
* @param {number} blockCount
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async resize(blockCount) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call resize of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
// Checks that the BAT is at least as big as the provided block count, if not, increases it and shift the blocks position
|
||||
await this.#vhd.ensureBatSize(blockCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes Block Allocation Table
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call flushMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
await this.#vhd.writeBlockAllocationTable()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {VhdFooter}
|
||||
*/
|
||||
getMetadata() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call getMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd.footer
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async mergeMetadata(childDisk) {
|
||||
const childDiskMetadata = childDisk.getMetadata()
|
||||
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call mergeMetadata of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.currentSize = childDiskMetadata.currentSize
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.diskGeometry = { ...childDiskMetadata.diskGeometry }
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.originalSize = childDiskMetadata.originalSize
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.timestamp = childDiskMetadata.timestamp
|
||||
// @ts-ignore
|
||||
this.#vhd.footer.uuid = childDiskMetadata.uuid
|
||||
|
||||
await this.#vhd.writeFooter()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
if (this.#isDifferencing === undefined) {
|
||||
throw new Error(`can't call isDifferencing of a RemoteVhdDisk before init`)
|
||||
}
|
||||
return this.#isDifferencing
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
if (isVhdAlias(newPath)) {
|
||||
const dataPath = await resolveVhdAlias(this.#handler, this.#path)
|
||||
|
||||
await this.#handler.unlink(this.#path)
|
||||
await this.#handler.unlink(newPath)
|
||||
|
||||
await VhdAbstract.createAlias(this.#handler, newPath, dataPath)
|
||||
|
||||
this.#path = newPath
|
||||
} else {
|
||||
try {
|
||||
await this.#handler.unlink(newPath)
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(newPath).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
await this.#handler.rename(this.#path, newPath)
|
||||
|
||||
this.#path = newPath
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes disk
|
||||
*/
|
||||
async unlink() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call unlink of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
await this.close()
|
||||
|
||||
if (isVhdAlias(this.#path)) {
|
||||
try {
|
||||
await this.#handler.unlink(await resolveVhdAlias(this.#handler, this.#path))
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(await resolveVhdAlias(this.#handler, this.#path)).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#handler.unlink(this.#path)
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EISDIR') {
|
||||
await this.#handler.rmtree(this.#path).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the disk is a VHD directory.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isDirectory() {
|
||||
if (this.#vhd === undefined) {
|
||||
throw new Error(`can't call isDirectory of a RemoteVhdDisk before init`)
|
||||
}
|
||||
|
||||
return this.#vhd instanceof VhdDirectory
|
||||
}
|
||||
}
|
||||
304
@xen-orchestra/backups/disks/RemoteVhdDiskChain.mjs
Normal file
304
@xen-orchestra/backups/disks/RemoteVhdDiskChain.mjs
Normal file
@@ -0,0 +1,304 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').VhdFooter} VhdFooter
|
||||
* @typedef {import('./RemoteVhdDisk.mjs').RemoteVhdDisk} RemoteVhdDisk
|
||||
* @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
|
||||
* @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
|
||||
*/
|
||||
|
||||
import { RemoteDisk } from './RemoteDisk.mjs'
|
||||
|
||||
export class RemoteVhdDiskChain extends RemoteDisk {
|
||||
/**
|
||||
* @type {RemoteVhdDisk[]}
|
||||
*/
|
||||
#disks
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#blockSize = 2 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#headerSize = 1024
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#footerSize = 512
|
||||
|
||||
/**
|
||||
* @type {number}
|
||||
*/
|
||||
#bitmapSize = 512
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {RemoteVhdDisk[]} params.disks
|
||||
*/
|
||||
constructor({ disks }) {
|
||||
super()
|
||||
this.#disks = disks
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the VHD chain
|
||||
* @param {Object} options
|
||||
* @param {boolean} options.force
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(options) {
|
||||
try {
|
||||
await Promise.all(this.#disks.map(disk => disk.init(options)))
|
||||
let parentUuid = ''
|
||||
for (const [index, disk] of this.#disks.entries()) {
|
||||
if (index !== 0) {
|
||||
if (!disk.isDifferencing()) {
|
||||
throw Object.assign(new Error("Can't init vhd directory with non differencing child disks"), {
|
||||
code: 'NOT_SUPPORTED',
|
||||
})
|
||||
}
|
||||
if (disk.getParentUuid() !== parentUuid) {
|
||||
throw Object.assign(new Error("Can't init vhd directory with incorrect parentage"), {
|
||||
code: 'NOT_SUPPORTED',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
parentUuid = disk.getUuid()
|
||||
}
|
||||
} catch (error) {
|
||||
await this.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the VHD.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async close() {
|
||||
await Promise.all(this.#disks.map(disk => disk.close()))
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getVirtualSize() {
|
||||
return this.#disks[this.#disks.length - 1].getVirtualSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} size
|
||||
*/
|
||||
getSizeOnDisk() {
|
||||
const batEntrySize = 4
|
||||
const sectorSize = 512
|
||||
const batSize = Math.ceil((this.getMaxBlockCount() * batEntrySize) / sectorSize) * sectorSize
|
||||
|
||||
return (
|
||||
this.#footerSize +
|
||||
this.#headerSize +
|
||||
batSize +
|
||||
this.getBlockIndexes().length * (this.#blockSize + this.#bitmapSize) +
|
||||
this.#footerSize
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number}
|
||||
*/
|
||||
getBlockSize() {
|
||||
return this.#disks[this.#disks.length - 1].getBlockSize()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getPath() {
|
||||
return this.#disks[this.#disks.length - 1].getPath()
|
||||
}
|
||||
|
||||
/**
|
||||
* Disk chains return an array of disk paths.
|
||||
*
|
||||
* @returns {string[]}
|
||||
*/
|
||||
getPaths() {
|
||||
return this.#disks.map(disk => disk.getPath())
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {string}
|
||||
*/
|
||||
getUuid() {
|
||||
return this.#disks[this.#disks.length - 1].getUuid()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<boolean>} canMergeConcurently
|
||||
*/
|
||||
async canMergeConcurently() {
|
||||
return this.isDirectory()
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {number} getMaxBlockCount
|
||||
*/
|
||||
getMaxBlockCount() {
|
||||
return this.#disks[this.#disks.length - 1].getMaxBlockCount()
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD contains a specific block.
|
||||
* @param {number} index
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasBlock(index) {
|
||||
for (const disk of this.#disks) {
|
||||
if (disk.hasBlock(index)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexes of all blocks in the VHD.
|
||||
* @returns {Array<number>}
|
||||
*/
|
||||
getBlockIndexes() {
|
||||
const indexes = new Set()
|
||||
for (const disk of this.#disks) {
|
||||
for (const index of disk.getBlockIndexes()) {
|
||||
indexes.add(index)
|
||||
}
|
||||
}
|
||||
return [...indexes]
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the parent non inizialized instance
|
||||
* @returns {RemoteDisk}
|
||||
*/
|
||||
instantiateParent() {
|
||||
return this.#disks[0].instantiateParent()
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a full block into this VHD.
|
||||
* @param {DiskBlock} diskBlock
|
||||
* @return {Promise<number>} blockSize
|
||||
*/
|
||||
async writeBlock(diskBlock) {
|
||||
throw new Error(`Can't write blocks into a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the VHD.
|
||||
* @param {number} index
|
||||
* @returns {Promise<DiskBlock>} diskBlock
|
||||
*/
|
||||
async readBlock(index) {
|
||||
const reversedDisks = this.#disks.slice().reverse()
|
||||
for (const disk of reversedDisks) {
|
||||
if (disk.hasBlock(index)) {
|
||||
return disk.readBlock(index)
|
||||
}
|
||||
}
|
||||
throw new Error(`Block ${index} not found in chain `)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a specific block from the child disk to copy/move it to this disk.
|
||||
* @param {RemoteDisk} childDisk
|
||||
* @param {number} index
|
||||
* @param {boolean} isResumingMerge
|
||||
* @returns {Promise<number>} blockSize
|
||||
*/
|
||||
async mergeBlock(childDisk, index, isResumingMerge) {
|
||||
throw new Error(`Can't merge block into a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a specific block path from the VHD directory disk.
|
||||
* @param {number} index
|
||||
* @returns {string} blockPath
|
||||
*/
|
||||
getBlockPath(index) {
|
||||
for (const disk of [...this.#disks].reverse()) {
|
||||
if (disk.hasBlock(index)) {
|
||||
return disk.getBlockPath(index)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Block ${index} not found in chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {VhdFooter}
|
||||
*/
|
||||
getMetadata() {
|
||||
return this.#disks[this.#disks.length - 1].getMetadata()
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async flushMetadata(childDisk) {
|
||||
throw new Error(`Can't flush metadata on a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {RemoteVhdDisk} childDisk
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
mergeMetadata(childDisk) {
|
||||
throw new Error(`Can't merge metadata on a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the VHD is a differencing disk.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isDifferencing() {
|
||||
return this.#disks[0].isDifferencing()
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract
|
||||
* Rename alias/disk
|
||||
* @param {string} newPath
|
||||
*/
|
||||
async rename(newPath) {
|
||||
throw new Error(`Can't rename a disk chain`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes all the disks
|
||||
*/
|
||||
async unlink() {
|
||||
for (const disk of this.#disks) {
|
||||
await disk.unlink()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if all the disks in the chain are VHD directories.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isDirectory() {
|
||||
for (const disk of this.#disks) {
|
||||
if (!(await disk.isDirectory())) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
49
@xen-orchestra/backups/disks/index.mjs
Normal file
49
@xen-orchestra/backups/disks/index.mjs
Normal file
@@ -0,0 +1,49 @@
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
export { RemoteDisk } from './RemoteDisk.mjs'
|
||||
export { openDiskChain } from './openDiskChain.mjs'
|
||||
|
||||
/**
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @returns {Promise<RemoteDisk>}
|
||||
*/
|
||||
export async function openDisk({ handler, path }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
return disk
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
* @returns {Promise<Disposable<RemoteDisk>>}
|
||||
*/
|
||||
export async function openDisposableDisk({ handler, path }) {
|
||||
const disk = new RemoteVhdDisk({ handler, path })
|
||||
await disk.init()
|
||||
return {
|
||||
value: disk,
|
||||
dispose: () => disk.close(),
|
||||
}
|
||||
}
|
||||
|
||||
const DISK_EXTENSIONS = ['.vhd']
|
||||
|
||||
/**
|
||||
* Returns true if the path points to a supported disk format.
|
||||
*
|
||||
* @param {FileAccessor} _handler - Remote file handler (reserved for future use)
|
||||
* @param {string} path - Path to check
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isDisk(_handler, path) {
|
||||
return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
|
||||
}
|
||||
@@ -2,12 +2,14 @@
|
||||
/**
|
||||
*
|
||||
* @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
|
||||
* @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
|
||||
*/
|
||||
import { DiskChain } from '@xen-orchestra/disk-transform'
|
||||
import { RemoteVhd } from './RemoteVhd.mjs'
|
||||
import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
|
||||
|
||||
import { defer } from 'golike-defer'
|
||||
/**
|
||||
* @param {any} $defer
|
||||
* @param {Object} params
|
||||
* @param {FileAccessor} params.handler
|
||||
* @param {string} params.path
|
||||
@@ -15,15 +17,18 @@ import { defer } from 'golike-defer'
|
||||
*/
|
||||
async function _openDiskChain($defer, { handler, path, until }) {
|
||||
let disk
|
||||
/**
|
||||
* @type {Array<RemoteDisk>}
|
||||
*/
|
||||
const disks = []
|
||||
$defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
|
||||
disk = new RemoteVhd({ handler, path })
|
||||
disk = new RemoteVhdDisk({ handler, path })
|
||||
|
||||
await disk.init()
|
||||
disks.push(disk)
|
||||
while (disk.isDifferencing()) {
|
||||
disk = await disk.openParent()
|
||||
if (disk.path === until) {
|
||||
if (disk.getPath() === until) {
|
||||
break
|
||||
}
|
||||
disks.unshift(disk)
|
||||
|
||||
116
@xen-orchestra/backups/docs/VM backups/incrementalReplication.md
Normal file
116
@xen-orchestra/backups/docs/VM backups/incrementalReplication.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# Incremental Replication (CR) - Target Selection
|
||||
|
||||
This document explains how incremental replication (Continuous Replication to a XAPI SR) selects its target VM and VDIs, and the resulting transfer modes.
|
||||
|
||||
## Overview
|
||||
|
||||
The goal is to maintain a replica VM on a target SR. On each run, the system decides:
|
||||
|
||||
1. **Target VM**: should we update an existing replicated VM, or create a new one?
|
||||
2. **Target VDIs**: for each disk, should we transfer a delta (incremental) or a full copy?
|
||||
|
||||
These two decisions are independent: we can create a new VM but still chain disks to existing VDIs from a previous replica, reducing transfer size.
|
||||
|
||||
## Key metadata (`other_config`)
|
||||
|
||||
Replicated VDI snapshots on the target SR are tagged with:
|
||||
|
||||
| Key | Purpose |
|
||||
| -------------------- | -------------------------------------------------- |
|
||||
| `xo:copy_of` | UUID of the source VDI this is a copy of |
|
||||
| `xo:backup:job` | Job ID that created it |
|
||||
| `xo:backup:vm` | Source VM UUID |
|
||||
| `xo:backup:schedule` | Schedule ID |
|
||||
| `xo:backup:sr` | Target SR UUID |
|
||||
| `xo:base_delta_vdi` | UUID of the source VDI used as base for this delta |
|
||||
|
||||
## Step-by-step flow
|
||||
|
||||
### 1. `_selectBaseVm` (runner)
|
||||
|
||||
Finds the last successfully exported snapshot VDIs on the source. Builds `baseUuidToSrcVdiUuid`: a map of `snapshotVdiUuid -> currentSrcVdiUuid` for each disk. Then calls `checkBaseVdis` on each writer with a shared copy of this map (`presentBaseVdis`).
|
||||
|
||||
Each writer removes entries from `presentBaseVdis` for disks it cannot handle as delta. Since writers are called sequentially on the same map, the result is an **intersection**: only disks that are valid across all writers survive. If any single writer (e.g. a remote writer) lacks a base for a disk, that disk is removed for everyone, and the runner will export it as full.
|
||||
|
||||
### 2. `checkBaseVdis` (writer)
|
||||
|
||||
Searches the target SR for VDI snapshots that match the source snapshot by `COPY_OF`, scoped to the current `JOB_ID` and `VM_UUID`. For each candidate, it validates:
|
||||
|
||||
1. **Active VDI exists**: the snapshot's `$snapshot_of` must resolve to a live VDI.
|
||||
2. **Single user VM**: the active VDI must be attached to exactly one non-control-domain VM (shared VDIs are skipped).
|
||||
3. **VM is blocked**: the VM must have `start` in `blocked_operations` (confirms it's a managed replica).
|
||||
4. **No data written since snapshot**: a diff between the active VDI and the snapshot must be empty (no blocks changed).
|
||||
|
||||
This produces two results stored as instance state:
|
||||
|
||||
- **`#baseVdisBySourceUuid`**: a `Map<sourceVdiUuid, activeVdi>` of validated disk links, reused later by `#decorateVmMetadata`.
|
||||
- **`_targetVmRef`**: the ref of the replicated VM to update, set only if **all** disks can chain to the same VM (`canChainToTargetVm`).
|
||||
|
||||
Any source VDI UUID not present in `#baseVdisBySourceUuid` is removed from `baseUuidToSrcVdi`, signaling the runner that no valid base exists for that disk.
|
||||
|
||||
### 3. `#decorateVmMetadata` (writer, called during transfer)
|
||||
|
||||
Sets `vdi.baseVdi` for each disk in the export by looking up `vdi.other_config[BASE_DELTA_VDI]` in `#baseVdisBySourceUuid`. This directly reuses the validated mapping from `checkBaseVdis` instead of re-querying the SR.
|
||||
|
||||
**Important**: a disk may arrive as a full export (no `BASE_DELTA_VDI`) even though this writer's `checkBaseVdis` had found a valid candidate for it. This happens when another writer in the job could not handle that disk as delta, causing the runner to remove it from the shared map and export it as full for all writers. In that case `BASE_DELTA_VDI` is absent, `#baseVdisBySourceUuid` is not consulted, and the disk is transferred as full.
|
||||
|
||||
### 4. `_transfer` (writer)
|
||||
|
||||
Calls `importIncrementalVm` with:
|
||||
|
||||
- `targetRef = this._targetVmRef` (may be `undefined` if no valid target VM)
|
||||
- Each VDI's `baseVdi` set by `#decorateVmMetadata` (may be `undefined` per-disk)
|
||||
|
||||
## Decision matrix
|
||||
|
||||
### Target VM selection
|
||||
|
||||
| All VDI diffs empty | VM blocked | `_targetVmRef` set | Result |
|
||||
| ---------------------- | ---------- | ------------------ | -------------------------------------------------------- |
|
||||
| Yes | Yes | Yes | **Update existing VM** (incremental into the same VM) |
|
||||
| No (some disk changed) | - | No | **Create new VM** (but may still chain individual disks) |
|
||||
| - | No | No | **Create new VM** (target VM was tampered with) |
|
||||
| No candidates found | - | No | **Create new VM** (first replication or target deleted) |
|
||||
|
||||
### Per-disk transfer mode
|
||||
|
||||
| `baseVdi` set | `BASE_DELTA_VDI` present | Result |
|
||||
| ------------- | ------------------------ | ---------------------------------------------------------------------------- |
|
||||
| Yes | Yes | **Delta transfer**: only changed blocks since the base are sent |
|
||||
| No | Yes | **Full transfer for this disk**: base was not found or not valid on target |
|
||||
| - | No | **Full transfer**: first replication of this disk (no base exists on source) |
|
||||
|
||||
### Combined scenarios
|
||||
|
||||
#### Scenario 1: Normal incremental run
|
||||
|
||||
All disks have valid bases on the target, diffs are empty. `_targetVmRef` is set. Transfer is fully incremental into the existing VM.
|
||||
|
||||
#### Scenario 2: One disk was modified on the target
|
||||
|
||||
Disk A's diff is empty, disk B has blocks written. `canChainToTargetVm = false`, so a new VM is created. But disk A still has a valid entry in `#baseVdisBySourceUuid`, so its transfer is incremental (chained to the existing active VDI). Disk B falls back to full.
|
||||
|
||||
#### Scenario 3: New disk added to source VM
|
||||
|
||||
Existing disks have valid bases. The new disk has no `BASE_DELTA_VDI`. Existing disks transfer as delta; the new disk transfers as full. If the existing disks' VM is valid, `_targetVmRef` is set and the new disk is added to that VM.
|
||||
|
||||
#### Scenario 4: Target VM deleted or first run
|
||||
|
||||
No snapshot candidates found. `#baseVdisBySourceUuid` is empty. All disks are full. A new VM is created.
|
||||
|
||||
#### Scenario 5: Disk valid on one writer but not another
|
||||
|
||||
A job replicates to both SR-A (XAPI writer) and Remote-B (remote writer). Disk X has a valid base on SR-A but the VHD chain is broken on Remote-B. Remote-B removes disk X from the shared `presentBaseVdis` map. The runner exports disk X as full. SR-A's `#decorateVmMetadata` receives a full disk (no `BASE_DELTA_VDI`), so `baseVdi` is `undefined` despite SR-A having a valid candidate locally. Both writers get a full transfer for that disk.
|
||||
|
||||
#### Scenario 6: Multiple jobs replicate the same VM to the same SR
|
||||
|
||||
`checkBaseVdis` scopes candidates by `JOB_ID` + `VM_UUID`, so each job only sees its own VDI snapshots. `#decorateVmMetadata` reuses this scoped mapping, avoiding cross-job conflicts.
|
||||
|
||||
## After transfer
|
||||
|
||||
Once the transfer completes:
|
||||
|
||||
1. `other_config` is updated on the target VM with the final timestamp.
|
||||
2. A snapshot of the target VM is taken (to serve as the base for the next delta).
|
||||
3. `other_config` is reset on the target VM (the snapshot retains the metadata).
|
||||
4. Old entries beyond `copyRetention` are destroyed during cleanup.
|
||||
@@ -18,6 +18,8 @@ function formatVmBackup(backup) {
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: 'xo-vm-backup',
|
||||
backupRepository: backup.backupRepositoryId,
|
||||
disks:
|
||||
backup.vhds === undefined
|
||||
? []
|
||||
@@ -38,17 +40,24 @@ function formatVmBackup(backup) {
|
||||
size: backup.size,
|
||||
timestamp: backup.timestamp,
|
||||
vm: {
|
||||
uuid: backup.vm.uuid,
|
||||
name_description: backup.vm.name_description,
|
||||
name_label: backup.vm.name_label,
|
||||
tags: backup.vm.tags,
|
||||
},
|
||||
|
||||
tags: backup.tags ?? [],
|
||||
differencingVhds,
|
||||
dynamicVhds,
|
||||
withMemory,
|
||||
}
|
||||
}
|
||||
|
||||
// format all backups as returned by RemoteAdapter#listAllVmBackups()
|
||||
export function formatVmBackups(backupsByVM) {
|
||||
return mapValues(backupsByVM, backups => backups.map(formatVmBackup))
|
||||
/**
|
||||
* format all backups as returned by RemoteAdapter#listAllVmBackups()
|
||||
* @param {Record<string, object[]>} backupsByVM
|
||||
* @param {string} backupRepositoryId
|
||||
* @returns {Record<string, object[]>}
|
||||
*/
|
||||
export function formatVmBackups(backupsByVM, backupRepositoryId) {
|
||||
return mapValues(backupsByVM, backups => backups.map(backup => formatVmBackup({ ...backup, backupRepositoryId })))
|
||||
}
|
||||
|
||||
@@ -8,32 +8,34 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.64.3",
|
||||
"version": "0.72.0",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "node --test",
|
||||
"test-integration": "node --test *.integ.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@iarna/toml": "^2.2.5",
|
||||
"@kldzj/stream-throttle": "^1.1.1",
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@vates/async-each": "^1.0.3",
|
||||
"@vates/cached-dns.lookup": "^1.0.0",
|
||||
"@vates/compose": "^2.1.0",
|
||||
"@vates/decorate-with": "^2.1.0",
|
||||
"@vates/disposable": "^0.1.6",
|
||||
"@vates/fuse-vhd": "^2.1.2",
|
||||
"@vates/generator-toolbox": "^1.0.4",
|
||||
"@vates/nbd-client": "^3.2.1",
|
||||
"@vates/generator-toolbox": "^1.1.2",
|
||||
"@vates/nbd-client": "^3.4.0",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/disk-transform": "^1.1.0",
|
||||
"@xen-orchestra/fs": "^4.6.3",
|
||||
"@xen-orchestra/log": "^0.7.1",
|
||||
"@xen-orchestra/qcow2": "^1.0.0",
|
||||
"@xen-orchestra/template": "^0.1.0",
|
||||
"@vates/task": "^0.7.0",
|
||||
"@xen-orchestra/async-map": "^0.1.3",
|
||||
"@xen-orchestra/disk-transform": "^1.2.3",
|
||||
"@xen-orchestra/fs": "^4.8.0",
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"@xen-orchestra/qcow2": "^1.3.0",
|
||||
"@xen-orchestra/template": "^0.1.1",
|
||||
"app-conf": "^3.0.0",
|
||||
"compare-versions": "^6.0.0",
|
||||
"d3-time-format": "^4.1.0",
|
||||
@@ -41,32 +43,36 @@
|
||||
"golike-defer": "^0.5.1",
|
||||
"human-format": "^1.2.0",
|
||||
"limit-concurrency-decorator": "^0.6.0",
|
||||
"lodash": "^4.17.20",
|
||||
"lodash": "^4.18.0",
|
||||
"moment-timezone": "^0.5.46",
|
||||
"ms": "^2.1.3",
|
||||
"node-zone": "^0.4.0",
|
||||
"parse-pairs": "^2.0.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"tar": "^6.1.15",
|
||||
"tar": "^7.5.3",
|
||||
"uuid": "^9.0.0",
|
||||
"value-matcher": "^0.2.0",
|
||||
"vhd-lib": "^4.14.1",
|
||||
"xen-api": "^4.7.4",
|
||||
"vhd-lib": "^4.16.0",
|
||||
"xen-api": "^4.7.7",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"fs-extra": "^11.1.0",
|
||||
"rimraf": "^5.0.1",
|
||||
"rimraf": "^6.0.1",
|
||||
"sinon": "^18.0.0",
|
||||
"tmp": "^0.2.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^8.4.1"
|
||||
"@xen-orchestra/xapi": "^8.7.2"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"exports": {
|
||||
"./disks": "./disks/index.mjs",
|
||||
"./*": "./*"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"preferGlobal": true,
|
||||
"dependencies": {
|
||||
"golike-defer": "^0.5.1",
|
||||
"xen-api": "^4.7.4"
|
||||
"xen-api": "^4.7.7"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/cron",
|
||||
"version": "1.0.6",
|
||||
"version": "1.0.7",
|
||||
"license": "ISC",
|
||||
"description": "Focused, well maintained, cron parser/scheduler",
|
||||
"keywords": [
|
||||
@@ -34,7 +34,7 @@
|
||||
"node": ">=8.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"lodash": "^4.17.4",
|
||||
"lodash": "^4.18.0",
|
||||
"moment-timezone": "^0.5.14"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
79
@xen-orchestra/disk-cli/.USAGE.md
Normal file
79
@xen-orchestra/disk-cli/.USAGE.md
Normal file
@@ -0,0 +1,79 @@
|
||||
```
|
||||
$ xo-disk-cli --help
|
||||
Usage: xo-disk-cli <command> <handler-url> <path> [options]
|
||||
|
||||
Commands:
|
||||
info Show disk info (virtual size, uid, parent uid, block size)
|
||||
list List all disks at a path and display their properties in a table
|
||||
transform Convert a disk to another format and write to stdout (raw | vhd | qcow2)
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
### `info`
|
||||
|
||||
Display metadata for a single disk file.
|
||||
|
||||
```
|
||||
xo-disk-cli info <handler-url> <disk-path>
|
||||
```
|
||||
|
||||
Output includes: UID, parent UID (for differencing disks), virtual size, and block size.
|
||||
|
||||
```
|
||||
$ xo-disk-cli info file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/snapshot.vhd
|
||||
Disk info: /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/snapshot.vhd
|
||||
UID: <uuid>
|
||||
Parent UID: <parent-uuid>
|
||||
Virtual size: 8.00 GiB (8589934592 bytes)
|
||||
Block size: 2.00 MiB (2097152 bytes)
|
||||
```
|
||||
|
||||
### `list`
|
||||
|
||||
List all disk files found at a directory path and display their properties in a table. Disks are sorted so that each child appears directly after its parent. When a disk's parent is the row immediately above, the parent UID column shows `↑` for quick chain health assessment.
|
||||
|
||||
```
|
||||
xo-disk-cli list <handler-url> <dir-path>
|
||||
```
|
||||
|
||||
```
|
||||
$ xo-disk-cli list file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/
|
||||
┌──────────────┬──────────────────────────────────────┬─────────────┬──────────────┬─────────────┬──────────────────────────────────────┐
|
||||
│ File │ UID │ Size on disk│ Virtual size │ Differencing │ Parent UID │
|
||||
├──────────────┼──────────────────────────────────────┼─────────────┼──────────────┼─────────────┼──────────────────────────────────────┤
|
||||
│ base.vhd │ xxxxxxxx-... │ 1.20 GiB │ 8.00 GiB │ no │ (none) │
|
||||
│ snapshot.vhd │ yyyyyyyy-... │ 128.00 MiB │ 8.00 GiB │ yes │ ↑ │
|
||||
└──────────────┴──────────────────────────────────────┴─────────────┴──────────────┴─────────────┴──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### `transform`
|
||||
|
||||
Convert a disk (or its full parent chain if differencing) to the specified format and write the result to stdout. Redirect to a file to save the output.
|
||||
|
||||
```
|
||||
xo-disk-cli transform <handler-url> <disk-path> <format>
|
||||
```
|
||||
|
||||
Supported formats:
|
||||
|
||||
- `raw` — flat binary image; absent blocks are written as zeros
|
||||
- `vhd` — VHD fixed/dynamic image
|
||||
- `qcow2` — QCOW2 image
|
||||
|
||||
```
|
||||
$ xo-disk-cli transform file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/snapshot.vhd raw > disk.img
|
||||
$ xo-disk-cli transform file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/snapshot.vhd qcow2 > disk.qcow2
|
||||
$ xo-disk-cli transform file:///mnt/backups /xo-vm-backups/<vm-uuid>/vdis/<vdi-uuid>/snapshot.vhd vhd > disk.vhd
|
||||
```
|
||||
|
||||
## Handler URLs
|
||||
|
||||
The `<handler-url>` argument identifies the remote storage backend. Examples:
|
||||
|
||||
| Backend | URL format |
|
||||
| ---------------- | ----------------------- |
|
||||
| Local filesystem | `file:///absolute/path` |
|
||||
| NFS | `nfs://host/export` |
|
||||
| SMB | `smb://host/share` |
|
||||
| S3 | `s3://bucket/prefix` |
|
||||
1
@xen-orchestra/disk-cli/.npmignore
Symbolic link
1
@xen-orchestra/disk-cli/.npmignore
Symbolic link
@@ -0,0 +1 @@
|
||||
../../scripts/npmignore
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user