mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-11-09 17:06:13 -06:00
feat(backups): validate VHD streams (#6770)
This commit is contained in:
32
@vates/diff/.USAGE.md
Normal file
32
@vates/diff/.USAGE.md
Normal file
@@ -0,0 +1,32 @@
|
||||
```js
|
||||
import diff from '@vates/diff'
|
||||
|
||||
diff('foo bar baz', 'Foo qux')
|
||||
// → [ 0, 'F', 4, 'qux', 7, '' ]
|
||||
//
|
||||
// Differences of the second string from the first one:
|
||||
// - at position 0, it contains `F`
|
||||
// - at position 4, it contains `qux`
|
||||
// - at position 7, it ends
|
||||
|
||||
diff('Foo qux', 'foo bar baz')
|
||||
// → [ 0, 'f', 4, 'bar', 7, ' baz' ]
|
||||
//
|
||||
// Differences of the second string from the first one:
|
||||
// - at position 0, it contains f`
|
||||
// - at position 4, it contains `bar`
|
||||
// - at position 7, it contains `baz`
|
||||
|
||||
// works with all collections that supports
|
||||
// - `.length`
|
||||
// - `collection[index]`
|
||||
// - `.slice(start, end)`
|
||||
//
|
||||
// which includes:
|
||||
// - arrays
|
||||
// - strings
|
||||
// - `Buffer`
|
||||
// - `TypedArray`
|
||||
diff([0, 1, 2], [3, 4])
|
||||
// → [ 0, [ 3, 4 ], 2, [] ]
|
||||
```
|
||||
1
@vates/diff/.npmignore
Symbolic link
1
@vates/diff/.npmignore
Symbolic link
@@ -0,0 +1 @@
|
||||
../../scripts/npmignore
|
||||
65
@vates/diff/README.md
Normal file
65
@vates/diff/README.md
Normal file
@@ -0,0 +1,65 @@
|
||||
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
|
||||
|
||||
# @vates/diff
|
||||
|
||||
[](https://npmjs.org/package/@vates/diff)  [](https://bundlephobia.com/result?p=@vates/diff) [](https://npmjs.org/package/@vates/diff)
|
||||
|
||||
> Computes differences between two arrays, buffers or strings
|
||||
|
||||
## Install
|
||||
|
||||
Installation of the [npm package](https://npmjs.org/package/@vates/diff):
|
||||
|
||||
```sh
|
||||
npm install --save @vates/diff
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```js
|
||||
import diff from '@vates/diff'
|
||||
|
||||
diff('foo bar baz', 'Foo qux')
|
||||
// → [ 0, 'F', 4, 'qux', 7, '' ]
|
||||
//
|
||||
// Differences of the second string from the first one:
|
||||
// - at position 0, it contains `F`
|
||||
// - at position 4, it contains `qux`
|
||||
// - at position 7, it ends
|
||||
|
||||
diff('Foo qux', 'foo bar baz')
|
||||
// → [ 0, 'f', 4, 'bar', 7, ' baz' ]
|
||||
//
|
||||
// Differences of the second string from the first one:
|
||||
// - at position 0, it contains f`
|
||||
// - at position 4, it contains `bar`
|
||||
// - at position 7, it contains `baz`
|
||||
|
||||
// works with all collections that supports
|
||||
// - `.length`
|
||||
// - `collection[index]`
|
||||
// - `.slice(start, end)`
|
||||
//
|
||||
// which includes:
|
||||
// - arrays
|
||||
// - strings
|
||||
// - `Buffer`
|
||||
// - `TypedArray`
|
||||
diff([0, 1, 2], [3, 4])
|
||||
// → [ 0, [ 3, 4 ], 2, [] ]
|
||||
```
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
the code.
|
||||
|
||||
You may:
|
||||
|
||||
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
|
||||
you've encountered;
|
||||
- fork and create a pull request.
|
||||
|
||||
## License
|
||||
|
||||
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)
|
||||
37
@vates/diff/index.js
Normal file
37
@vates/diff/index.js
Normal file
@@ -0,0 +1,37 @@
|
||||
'use strict'
|
||||
|
||||
/**
|
||||
* Compare two data arrays, buffers or strings and invoke the provided callback function for each difference.
|
||||
*
|
||||
* @template {Array|Buffer|string} T
|
||||
* @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
|
||||
* @param {T} data2 - The second data array or buffer to compare.
|
||||
* @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
|
||||
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
|
||||
*/
|
||||
module.exports = function diff(data1, data2, cb) {
|
||||
let result
|
||||
if (cb === undefined) {
|
||||
result = []
|
||||
cb = result.push.bind(result)
|
||||
}
|
||||
|
||||
const n1 = data1.length
|
||||
const n2 = data2.length
|
||||
const n = Math.min(n1, n2)
|
||||
for (let i = 0; i < n; ++i) {
|
||||
if (data1[i] !== data2[i]) {
|
||||
let j = i + 1
|
||||
while (j < n && data1[j] !== data2[j]) {
|
||||
++j
|
||||
}
|
||||
cb(i, data2.slice(i, j))
|
||||
i = j
|
||||
}
|
||||
}
|
||||
if (n1 !== n2) {
|
||||
cb(n, n1 < n2 ? data2.slice(n) : data2.slice(0, 0))
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
51
@vates/diff/index.test.js
Normal file
51
@vates/diff/index.test.js
Normal file
@@ -0,0 +1,51 @@
|
||||
'use strict'
|
||||
|
||||
const assert = require('node:assert/strict')
|
||||
const test = require('test')
|
||||
|
||||
const diff = require('./index.js')
|
||||
|
||||
test('data of equal length', function () {
|
||||
const data1 = 'foo bar baz'
|
||||
const data2 = 'baz bar foo'
|
||||
assert.deepEqual(diff(data1, data2), [0, 'baz', 8, 'foo'])
|
||||
})
|
||||
|
||||
test('data1 is longer', function () {
|
||||
const data1 = 'foo bar'
|
||||
const data2 = 'foo'
|
||||
assert.deepEqual(diff(data1, data2), [3, ''])
|
||||
})
|
||||
|
||||
test('data2 is longer', function () {
|
||||
const data1 = 'foo'
|
||||
const data2 = 'foo bar'
|
||||
assert.deepEqual(diff(data1, data2), [3, ' bar'])
|
||||
})
|
||||
|
||||
test('with arrays', function () {
|
||||
const data1 = 'foo bar baz'.split('')
|
||||
const data2 = 'baz bar foo'.split('')
|
||||
assert.deepEqual(diff(data1, data2), [0, 'baz'.split(''), 8, 'foo'.split('')])
|
||||
})
|
||||
|
||||
test('with buffers', function () {
|
||||
const data1 = Buffer.from('foo bar baz')
|
||||
const data2 = Buffer.from('baz bar foo')
|
||||
assert.deepEqual(diff(data1, data2), [0, Buffer.from('baz'), 8, Buffer.from('foo')])
|
||||
})
|
||||
|
||||
test('cb param', function () {
|
||||
const data1 = 'foo bar baz'
|
||||
const data2 = 'baz bar foo'
|
||||
|
||||
const calls = []
|
||||
const cb = (...args) => calls.push(args)
|
||||
|
||||
diff(data1, data2, cb)
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
[0, 'baz'],
|
||||
[8, 'foo'],
|
||||
])
|
||||
})
|
||||
36
@vates/diff/package.json
Normal file
36
@vates/diff/package.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@vates/diff",
|
||||
"description": "Computes differences between two arrays, buffers or strings",
|
||||
"keywords": [
|
||||
"array",
|
||||
"binary",
|
||||
"buffer",
|
||||
"diff",
|
||||
"differences",
|
||||
"string"
|
||||
],
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/diff",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@vates/diff",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "0.0.0",
|
||||
"engines": {
|
||||
"node": ">=8.10"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "node--test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"test": "^3.3.0"
|
||||
}
|
||||
}
|
||||
42
@vates/stream-reader/.USAGE.md
Normal file
42
@vates/stream-reader/.USAGE.md
Normal file
@@ -0,0 +1,42 @@
|
||||
```js
|
||||
import StreamReader from '@vates/stream-reader'
|
||||
|
||||
const reader = new StreamReader(stream)
|
||||
```
|
||||
|
||||
### `.read([size])`
|
||||
|
||||
- returns the next available chunk of data
|
||||
- like `stream.read()`, a number of bytes can be specified
|
||||
- returns with less data than expected if stream has ended
|
||||
- returns `null` if the stream has ended and no data has been read
|
||||
|
||||
```js
|
||||
const chunk = await reader.read(512)
|
||||
```
|
||||
|
||||
### `.readStrict([size])`
|
||||
|
||||
Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
|
||||
|
||||
```js
|
||||
const chunk = await reader.readStrict(512)
|
||||
```
|
||||
|
||||
### `.skip(size)`
|
||||
|
||||
Skips a given number of bytes from a stream.
|
||||
|
||||
Returns the number of bytes actually skipped, which may be less than the requested size if the stream has ended.
|
||||
|
||||
```js
|
||||
const bytesSkipped = await reader.skip(2 * 1024 * 1024 * 1024)
|
||||
```
|
||||
|
||||
### `.skipStrict(size)`
|
||||
|
||||
Skips a given number of bytes from a stream and throws if the stream ended before enough stream has been skipped.
|
||||
|
||||
```js
|
||||
await reader.skipStrict(2 * 1024 * 1024 * 1024)
|
||||
```
|
||||
1
@vates/stream-reader/.npmignore
Symbolic link
1
@vates/stream-reader/.npmignore
Symbolic link
@@ -0,0 +1 @@
|
||||
../../scripts/npmignore
|
||||
75
@vates/stream-reader/README.md
Normal file
75
@vates/stream-reader/README.md
Normal file
@@ -0,0 +1,75 @@
|
||||
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
|
||||
|
||||
# @vates/stream-reader
|
||||
|
||||
[](https://npmjs.org/package/@vates/stream-reader)  [](https://bundlephobia.com/result?p=@vates/stream-reader) [](https://npmjs.org/package/@vates/stream-reader)
|
||||
|
||||
> Efficiently reads and skips chunks of a given size in a stream
|
||||
|
||||
## Install
|
||||
|
||||
Installation of the [npm package](https://npmjs.org/package/@vates/stream-reader):
|
||||
|
||||
```sh
|
||||
npm install --save @vates/stream-reader
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```js
|
||||
import StreamReader from '@vates/stream-reader'
|
||||
|
||||
const reader = new StreamReader(stream)
|
||||
```
|
||||
|
||||
### `.read([size])`
|
||||
|
||||
- returns the next available chunk of data
|
||||
- like `stream.read()`, a number of bytes can be specified
|
||||
- returns with less data than expected if stream has ended
|
||||
- returns `null` if the stream has ended and no data has been read
|
||||
|
||||
```js
|
||||
const chunk = await reader.read(512)
|
||||
```
|
||||
|
||||
### `.readStrict([size])`
|
||||
|
||||
Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
|
||||
|
||||
```js
|
||||
const chunk = await reader.readStrict(512)
|
||||
```
|
||||
|
||||
### `.skip(size)`
|
||||
|
||||
Skips a given number of bytes from a stream.
|
||||
|
||||
Returns the number of bytes actually skipped, which may be less than the requested size if the stream has ended.
|
||||
|
||||
```js
|
||||
const bytesSkipped = await reader.skip(2 * 1024 * 1024 * 1024)
|
||||
```
|
||||
|
||||
### `.skipStrict(size)`
|
||||
|
||||
Skips a given number of bytes from a stream and throws if the stream ended before enough stream has been skipped.
|
||||
|
||||
```js
|
||||
await reader.skipStrict(2 * 1024 * 1024 * 1024)
|
||||
```
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
the code.
|
||||
|
||||
You may:
|
||||
|
||||
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
|
||||
you've encountered;
|
||||
- fork and create a pull request.
|
||||
|
||||
## License
|
||||
|
||||
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)
|
||||
123
@vates/stream-reader/index.js
Normal file
123
@vates/stream-reader/index.js
Normal file
@@ -0,0 +1,123 @@
|
||||
'use strict'
|
||||
|
||||
const assert = require('node:assert')
|
||||
const { finished, Readable } = require('node:stream')
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
// Inspired by https://github.com/nodejs/node/blob/85705a47958c9ae5dbaa1f57456db19bdefdc494/lib/internal/streams/readable.js#L1107
|
||||
class StreamReader {
|
||||
#ended = false
|
||||
#error
|
||||
#executor = resolve => {
|
||||
this.#resolve = resolve
|
||||
}
|
||||
#stream
|
||||
#resolve = noop
|
||||
|
||||
constructor(stream) {
|
||||
stream = typeof stream.pipe === 'function' ? stream : Readable.from(stream)
|
||||
|
||||
this.#stream = stream
|
||||
|
||||
stream.on('readable', () => this.#resolve())
|
||||
|
||||
finished(stream, { writable: false }, error => {
|
||||
this.#error = error
|
||||
this.#ended = true
|
||||
this.#resolve()
|
||||
})
|
||||
}
|
||||
|
||||
async read(size) {
|
||||
if (size !== undefined) {
|
||||
assert(size > 0)
|
||||
}
|
||||
|
||||
do {
|
||||
if (this.#ended) {
|
||||
if (this.#error) {
|
||||
throw this.#error
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
const value = this.#stream.read(size)
|
||||
if (value !== null) {
|
||||
return value
|
||||
}
|
||||
|
||||
await new Promise(this.#executor)
|
||||
} while (true)
|
||||
}
|
||||
|
||||
async readStrict(size) {
|
||||
const chunk = await this.read(size)
|
||||
if (chunk === null) {
|
||||
throw new Error('stream has ended without data')
|
||||
}
|
||||
|
||||
if (size !== undefined && chunk.length !== size) {
|
||||
const error = new Error(`stream has ended with not enough data (actual: ${chunk.length}, expected: ${size})`)
|
||||
Object.defineProperties(error, {
|
||||
chunk: {
|
||||
value: chunk,
|
||||
},
|
||||
})
|
||||
throw error
|
||||
}
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
async skip(size) {
|
||||
if (size === 0) {
|
||||
return size
|
||||
}
|
||||
|
||||
let toSkip = size
|
||||
do {
|
||||
if (this.#ended) {
|
||||
if (this.#error) {
|
||||
throw this.#error
|
||||
}
|
||||
return size - toSkip
|
||||
}
|
||||
|
||||
const data = this.#stream.read()
|
||||
if (data !== null) {
|
||||
toSkip -= data === null ? 0 : data.length
|
||||
if (toSkip > 0) {
|
||||
// continue to read
|
||||
} else {
|
||||
// if more than wanted has been read, push back the rest
|
||||
if (toSkip < 0) {
|
||||
this.#stream.unshift(data.slice(toSkip))
|
||||
}
|
||||
|
||||
return size
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise(this.#executor)
|
||||
} while (true)
|
||||
}
|
||||
|
||||
async skipStrict(size) {
|
||||
const bytesSkipped = await this.skip(size)
|
||||
if (bytesSkipped !== size) {
|
||||
const error = new Error(`stream has ended with not enough data (actual: ${bytesSkipped}, expected: ${size})`)
|
||||
error.bytesSkipped = bytesSkipped
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StreamReader.prototype[Symbol.asyncIterator] = async function* asyncIterator() {
|
||||
let chunk
|
||||
while ((chunk = await this.read()) !== null) {
|
||||
yield chunk
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = StreamReader
|
||||
141
@vates/stream-reader/index.test.js
Normal file
141
@vates/stream-reader/index.test.js
Normal file
@@ -0,0 +1,141 @@
|
||||
'use strict'
|
||||
|
||||
const { describe, it } = require('test')
|
||||
const assert = require('node:assert').strict
|
||||
|
||||
const { Readable } = require('stream')
|
||||
|
||||
const StreamReader = require('./index.js')
|
||||
|
||||
const makeStream = it => Readable.from(it, { objectMode: false })
|
||||
makeStream.obj = Readable.from
|
||||
|
||||
const rejectionOf = promise =>
|
||||
promise.then(
|
||||
value => {
|
||||
throw value
|
||||
},
|
||||
error => error
|
||||
)
|
||||
|
||||
const makeErrorTests = method => {
|
||||
it('rejects if the stream errors', async () => {
|
||||
const error = new Error()
|
||||
const stream = makeStream([])
|
||||
|
||||
const pError = rejectionOf(new StreamReader(stream)[method](10))
|
||||
stream.destroy(error)
|
||||
|
||||
assert.strict(await pError, error)
|
||||
})
|
||||
|
||||
it('rejects if the stream has already errored', async () => {
|
||||
const error = new Error()
|
||||
const stream = makeStream([])
|
||||
|
||||
await new Promise(resolve => {
|
||||
stream.once('error', resolve).destroy(error)
|
||||
})
|
||||
|
||||
assert.strict(await rejectionOf(new StreamReader(stream)[method](10)), error)
|
||||
})
|
||||
}
|
||||
|
||||
describe('read()', () => {
|
||||
it('rejects if size is less than or equal to 0', async () => {
|
||||
const error = await rejectionOf(new StreamReader(makeStream([])).read(0))
|
||||
assert.strictEqual(error.code, 'ERR_ASSERTION')
|
||||
})
|
||||
|
||||
it('returns null if stream is empty', async () => {
|
||||
assert.strictEqual(await new StreamReader(makeStream([])).read(), null)
|
||||
})
|
||||
|
||||
makeErrorTests('read')
|
||||
|
||||
it('returns null if the stream is already ended', async () => {
|
||||
const reader = new StreamReader(makeStream([]))
|
||||
|
||||
await reader.read()
|
||||
|
||||
assert.strictEqual(await reader.read(), null)
|
||||
})
|
||||
|
||||
describe('with binary stream', () => {
|
||||
it('returns the first chunk of data', async () => {
|
||||
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(), Buffer.from('foo'))
|
||||
})
|
||||
|
||||
it('returns a chunk of the specified size (smaller than first)', async () => {
|
||||
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(2), Buffer.from('fo'))
|
||||
})
|
||||
|
||||
it('returns a chunk of the specified size (larger than first)', async () => {
|
||||
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(4), Buffer.from('foob'))
|
||||
})
|
||||
|
||||
it('returns less data if stream ends', async () => {
|
||||
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(10), Buffer.from('foobar'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('with object stream', () => {
|
||||
it('returns the first chunk of data verbatim', async () => {
|
||||
const chunks = [{}, {}]
|
||||
assert.strictEqual(await new StreamReader(makeStream.obj(chunks)).read(), chunks[0])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('readStrict()', function () {
|
||||
it('throws if stream is empty', async () => {
|
||||
const error = await rejectionOf(new StreamReader(makeStream([])).readStrict())
|
||||
assert(error instanceof Error)
|
||||
assert.strictEqual(error.message, 'stream has ended without data')
|
||||
assert.strictEqual(error.chunk, undefined)
|
||||
})
|
||||
|
||||
it('throws if stream ends with not enough data', async () => {
|
||||
const error = await rejectionOf(new StreamReader(makeStream(['foo', 'bar'])).readStrict(10))
|
||||
assert(error instanceof Error)
|
||||
assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 6, expected: 10)')
|
||||
assert.deepEqual(error.chunk, Buffer.from('foobar'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('skip()', function () {
|
||||
makeErrorTests('skip')
|
||||
|
||||
it('returns 0 if size is 0', async () => {
|
||||
assert.strictEqual(await new StreamReader(makeStream(['foo'])).skip(0), 0)
|
||||
})
|
||||
|
||||
it('returns 0 if the stream is already ended', async () => {
|
||||
const reader = new StreamReader(makeStream([]))
|
||||
|
||||
await reader.read()
|
||||
|
||||
assert.strictEqual(await reader.skip(10), 0)
|
||||
})
|
||||
|
||||
it('skips a number of bytes', async () => {
|
||||
const reader = new StreamReader(makeStream('foo bar'))
|
||||
|
||||
assert.strictEqual(await reader.skip(4), 4)
|
||||
assert.deepEqual(await reader.read(4), Buffer.from('bar'))
|
||||
})
|
||||
|
||||
it('returns less size if stream ends', async () => {
|
||||
assert.deepEqual(await new StreamReader(makeStream('foo bar')).skip(10), 7)
|
||||
})
|
||||
})
|
||||
|
||||
describe('skipStrict()', function () {
|
||||
it('throws if stream ends with not enough data', async () => {
|
||||
const error = await rejectionOf(new StreamReader(makeStream('foo bar')).skipStrict(10))
|
||||
|
||||
assert(error instanceof Error)
|
||||
assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 7, expected: 10)')
|
||||
assert.deepEqual(error.bytesSkipped, 7)
|
||||
})
|
||||
})
|
||||
39
@vates/stream-reader/package.json
Normal file
39
@vates/stream-reader/package.json
Normal file
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@vates/stream-reader",
|
||||
"description": "Efficiently reads and skips chunks of a given size in a stream",
|
||||
"keywords": [
|
||||
"async",
|
||||
"chunk",
|
||||
"data",
|
||||
"node",
|
||||
"promise",
|
||||
"read",
|
||||
"reader",
|
||||
"skip",
|
||||
"stream"
|
||||
],
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/stream-reader",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@vates/stream-reader",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "0.0.0",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public",
|
||||
"test": "node--test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"test": "^3.3.0"
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ const DEFAULT_VM_SETTINGS = {
|
||||
timeout: 0,
|
||||
useNbd: false,
|
||||
unconditionalSnapshot: false,
|
||||
validateVhdStreams: true,
|
||||
vmTimeout: 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -6,11 +6,13 @@ const groupBy = require('lodash/groupBy.js')
|
||||
const ignoreErrors = require('promise-toolbox/ignoreErrors')
|
||||
const keyBy = require('lodash/keyBy.js')
|
||||
const mapValues = require('lodash/mapValues.js')
|
||||
const vhdStreamValidator = require('vhd-lib/vhdStreamValidator.js')
|
||||
const { asyncMap } = require('@xen-orchestra/async-map')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { decorateMethodsWith } = require('@vates/decorate-with')
|
||||
const { defer } = require('golike-defer')
|
||||
const { formatDateTime } = require('@xen-orchestra/xapi')
|
||||
const { pipeline } = require('node:stream')
|
||||
|
||||
const { DeltaBackupWriter } = require('./writers/DeltaBackupWriter.js')
|
||||
const { DeltaReplicationWriter } = require('./writers/DeltaReplicationWriter.js')
|
||||
@@ -44,6 +46,8 @@ const forkDeltaExport = deltaExport =>
|
||||
},
|
||||
})
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
class VmBackup {
|
||||
constructor({
|
||||
config,
|
||||
@@ -251,6 +255,11 @@ class VmBackup {
|
||||
Task.info('Transfer data using NBD')
|
||||
}
|
||||
const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
|
||||
|
||||
if (this._settings.validateVhdStreams) {
|
||||
deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
|
||||
}
|
||||
|
||||
deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
|
||||
|
||||
const timestamp = Date.now()
|
||||
|
||||
@@ -34,7 +34,11 @@
|
||||
|
||||
<!--packages-start-->
|
||||
|
||||
- @vates/diff minor
|
||||
- @vates/read-chunk patch
|
||||
- @vates/stream-reader minor
|
||||
- @xen-orchestra/backups minor
|
||||
- vhd-lib minor
|
||||
- xo-web patch
|
||||
- xo-server minor
|
||||
- xo-server-usage-report patch
|
||||
|
||||
@@ -12,11 +12,13 @@
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": ">=16"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^1.0.0",
|
||||
"@vates/diff": "^0.0.0",
|
||||
"@vates/read-chunk": "^1.1.0",
|
||||
"@vates/stream-reader": "^0.0.0",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/fs": "^3.3.4",
|
||||
"@xen-orchestra/log": "^0.6.0",
|
||||
|
||||
79
packages/vhd-lib/vhdStreamValidator.js
Normal file
79
packages/vhd-lib/vhdStreamValidator.js
Normal file
@@ -0,0 +1,79 @@
|
||||
'use strict'
|
||||
|
||||
const diff = require('@vates/diff')
|
||||
const StreamReader = require('@vates/stream-reader')
|
||||
|
||||
const { FOOTER_SIZE, HEADER_SIZE } = require('./_constants.js')
|
||||
const { unpackFooter, unpackHeader } = require('./Vhd/_utils.js')
|
||||
|
||||
// Async generator that can be in a pipeline to check the header and footers of a VHD stream.
|
||||
//
|
||||
// ```js
|
||||
// await pipeline(createVhdStream(), vhdStreamValidator, createOutputStream())
|
||||
// ```
|
||||
module.exports = async function* vhdStreamValidator(source) {
|
||||
let offset = 0
|
||||
try {
|
||||
const reader = new StreamReader(source)
|
||||
|
||||
const footer1 = await reader.readStrict(FOOTER_SIZE)
|
||||
yield footer1
|
||||
|
||||
// unpack and validate footer
|
||||
unpackFooter(footer1)
|
||||
|
||||
offset += footer1.length
|
||||
|
||||
const header = await reader.readStrict(HEADER_SIZE)
|
||||
yield header
|
||||
|
||||
// unpack and validate header
|
||||
unpackHeader(header)
|
||||
|
||||
offset += header.length
|
||||
|
||||
let last
|
||||
while (true) {
|
||||
const chunk = await reader.read(FOOTER_SIZE)
|
||||
|
||||
if (chunk === null) {
|
||||
// the end has been reached, `last` contains the footer
|
||||
break
|
||||
}
|
||||
|
||||
yield chunk
|
||||
|
||||
offset += chunk.length
|
||||
|
||||
if (chunk.length < FOOTER_SIZE) {
|
||||
// the end has been reached, the footer is part in `last` and part in `chunk`
|
||||
last = Buffer.concat([last.slice(-(FOOTER_SIZE - chunk.length)), chunk])
|
||||
break
|
||||
}
|
||||
|
||||
last = chunk
|
||||
}
|
||||
|
||||
// offset is the position at which the footer was read, not the position after
|
||||
offset -= last.length
|
||||
|
||||
// do not use `assert.strictDeepEqual` as it does not provide a compact (for buffers) or complete (for strings) result
|
||||
//
|
||||
// - save only expected data and diff
|
||||
// - save expected buffer as hex string for compacity and readability
|
||||
// - save diff as an array of differences represented as hex strings
|
||||
if (!last.equals(footer1)) {
|
||||
const error = new Error('footer1 !== footer2')
|
||||
error.diff = []
|
||||
diff(footer1, last, (i, buf) => {
|
||||
error.diff.push(i, buf.toString('hex'))
|
||||
})
|
||||
error.expected = footer1.toString('hex')
|
||||
|
||||
throw error
|
||||
}
|
||||
} catch (error) {
|
||||
error.offset = offset
|
||||
throw error
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user