feat(backups): validate VHD streams (#6770)

This commit is contained in:
Julien Fontanet
2023-04-12 12:05:55 +02:00
committed by GitHub
parent 61f1316c42
commit 68b2c287eb
17 changed files with 739 additions and 1 deletions

32
@vates/diff/.USAGE.md Normal file
View File

@@ -0,0 +1,32 @@
```js
import diff from '@vates/diff'
diff('foo bar baz', 'Foo qux')
// → [ 0, 'F', 4, 'qux', 7, '' ]
//
// Differences of the second string from the first one:
// - at position 0, it contains `F`
// - at position 4, it contains `qux`
// - at position 7, it ends
diff('Foo qux', 'foo bar baz')
// → [ 0, 'f', 4, 'bar', 7, ' baz' ]
//
// Differences of the second string from the first one:
// - at position 0, it contains f`
// - at position 4, it contains `bar`
// - at position 7, it contains `baz`
// works with all collections that supports
// - `.length`
// - `collection[index]`
// - `.slice(start, end)`
//
// which includes:
// - arrays
// - strings
// - `Buffer`
// - `TypedArray`
diff([0, 1, 2], [3, 4])
// → [ 0, [ 3, 4 ], 2, [] ]
```

1
@vates/diff/.npmignore Symbolic link
View File

@@ -0,0 +1 @@
../../scripts/npmignore

65
@vates/diff/README.md Normal file
View File

@@ -0,0 +1,65 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/diff
[![Package Version](https://badgen.net/npm/v/@vates/diff)](https://npmjs.org/package/@vates/diff) ![License](https://badgen.net/npm/license/@vates/diff) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/diff)](https://bundlephobia.com/result?p=@vates/diff) [![Node compatibility](https://badgen.net/npm/node/@vates/diff)](https://npmjs.org/package/@vates/diff)
> Computes differences between two arrays, buffers or strings
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/diff):
```sh
npm install --save @vates/diff
```
## Usage
```js
import diff from '@vates/diff'
diff('foo bar baz', 'Foo qux')
// → [ 0, 'F', 4, 'qux', 7, '' ]
//
// Differences of the second string from the first one:
// - at position 0, it contains `F`
// - at position 4, it contains `qux`
// - at position 7, it ends
diff('Foo qux', 'foo bar baz')
// → [ 0, 'f', 4, 'bar', 7, ' baz' ]
//
// Differences of the second string from the first one:
// - at position 0, it contains f`
// - at position 4, it contains `bar`
// - at position 7, it contains `baz`
// works with all collections that supports
// - `.length`
// - `collection[index]`
// - `.slice(start, end)`
//
// which includes:
// - arrays
// - strings
// - `Buffer`
// - `TypedArray`
diff([0, 1, 2], [3, 4])
// → [ 0, [ 3, 4 ], 2, [] ]
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

37
@vates/diff/index.js Normal file
View File

@@ -0,0 +1,37 @@
'use strict'
/**
* Compare two data arrays, buffers or strings and invoke the provided callback function for each difference.
*
* @template {Array|Buffer|string} T
* @param {Array|Buffer|string} data1 - The first data array or buffer to compare.
* @param {T} data2 - The second data array or buffer to compare.
* @param {(index: number, diff: T) => void} [cb] - The callback function to invoke for each difference. If not provided, an array of differences will be returned.
* @returns {Array<number|T>|undefined} - An array of differences if no callback is provided, otherwise undefined.
*/
module.exports = function diff(data1, data2, cb) {
let result
if (cb === undefined) {
result = []
cb = result.push.bind(result)
}
const n1 = data1.length
const n2 = data2.length
const n = Math.min(n1, n2)
for (let i = 0; i < n; ++i) {
if (data1[i] !== data2[i]) {
let j = i + 1
while (j < n && data1[j] !== data2[j]) {
++j
}
cb(i, data2.slice(i, j))
i = j
}
}
if (n1 !== n2) {
cb(n, n1 < n2 ? data2.slice(n) : data2.slice(0, 0))
}
return result
}

51
@vates/diff/index.test.js Normal file
View File

@@ -0,0 +1,51 @@
'use strict'
const assert = require('node:assert/strict')
const test = require('test')
const diff = require('./index.js')
test('data of equal length', function () {
const data1 = 'foo bar baz'
const data2 = 'baz bar foo'
assert.deepEqual(diff(data1, data2), [0, 'baz', 8, 'foo'])
})
test('data1 is longer', function () {
const data1 = 'foo bar'
const data2 = 'foo'
assert.deepEqual(diff(data1, data2), [3, ''])
})
test('data2 is longer', function () {
const data1 = 'foo'
const data2 = 'foo bar'
assert.deepEqual(diff(data1, data2), [3, ' bar'])
})
test('with arrays', function () {
const data1 = 'foo bar baz'.split('')
const data2 = 'baz bar foo'.split('')
assert.deepEqual(diff(data1, data2), [0, 'baz'.split(''), 8, 'foo'.split('')])
})
test('with buffers', function () {
const data1 = Buffer.from('foo bar baz')
const data2 = Buffer.from('baz bar foo')
assert.deepEqual(diff(data1, data2), [0, Buffer.from('baz'), 8, Buffer.from('foo')])
})
test('cb param', function () {
const data1 = 'foo bar baz'
const data2 = 'baz bar foo'
const calls = []
const cb = (...args) => calls.push(args)
diff(data1, data2, cb)
assert.deepEqual(calls, [
[0, 'baz'],
[8, 'foo'],
])
})

36
@vates/diff/package.json Normal file
View File

@@ -0,0 +1,36 @@
{
"private": false,
"name": "@vates/diff",
"description": "Computes differences between two arrays, buffers or strings",
"keywords": [
"array",
"binary",
"buffer",
"diff",
"differences",
"string"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/diff",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/diff",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.0.0",
"engines": {
"node": ">=8.10"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
},
"devDependencies": {
"test": "^3.3.0"
}
}

View File

@@ -0,0 +1,42 @@
```js
import StreamReader from '@vates/stream-reader'
const reader = new StreamReader(stream)
```
### `.read([size])`
- returns the next available chunk of data
- like `stream.read()`, a number of bytes can be specified
- returns with less data than expected if stream has ended
- returns `null` if the stream has ended and no data has been read
```js
const chunk = await reader.read(512)
```
### `.readStrict([size])`
Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
```js
const chunk = await reader.readStrict(512)
```
### `.skip(size)`
Skips a given number of bytes from a stream.
Returns the number of bytes actually skipped, which may be less than the requested size if the stream has ended.
```js
const bytesSkipped = await reader.skip(2 * 1024 * 1024 * 1024)
```
### `.skipStrict(size)`
Skips a given number of bytes from a stream and throws if the stream ended before enough stream has been skipped.
```js
await reader.skipStrict(2 * 1024 * 1024 * 1024)
```

View File

@@ -0,0 +1 @@
../../scripts/npmignore

View File

@@ -0,0 +1,75 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/stream-reader
[![Package Version](https://badgen.net/npm/v/@vates/stream-reader)](https://npmjs.org/package/@vates/stream-reader) ![License](https://badgen.net/npm/license/@vates/stream-reader) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/stream-reader)](https://bundlephobia.com/result?p=@vates/stream-reader) [![Node compatibility](https://badgen.net/npm/node/@vates/stream-reader)](https://npmjs.org/package/@vates/stream-reader)
> Efficiently reads and skips chunks of a given size in a stream
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/stream-reader):
```sh
npm install --save @vates/stream-reader
```
## Usage
```js
import StreamReader from '@vates/stream-reader'
const reader = new StreamReader(stream)
```
### `.read([size])`
- returns the next available chunk of data
- like `stream.read()`, a number of bytes can be specified
- returns with less data than expected if stream has ended
- returns `null` if the stream has ended and no data has been read
```js
const chunk = await reader.read(512)
```
### `.readStrict([size])`
Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
```js
const chunk = await reader.readStrict(512)
```
### `.skip(size)`
Skips a given number of bytes from a stream.
Returns the number of bytes actually skipped, which may be less than the requested size if the stream has ended.
```js
const bytesSkipped = await reader.skip(2 * 1024 * 1024 * 1024)
```
### `.skipStrict(size)`
Skips a given number of bytes from a stream and throws if the stream ended before enough stream has been skipped.
```js
await reader.skipStrict(2 * 1024 * 1024 * 1024)
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -0,0 +1,123 @@
'use strict'
const assert = require('node:assert')
const { finished, Readable } = require('node:stream')
const noop = Function.prototype
// Inspired by https://github.com/nodejs/node/blob/85705a47958c9ae5dbaa1f57456db19bdefdc494/lib/internal/streams/readable.js#L1107
class StreamReader {
#ended = false
#error
#executor = resolve => {
this.#resolve = resolve
}
#stream
#resolve = noop
constructor(stream) {
stream = typeof stream.pipe === 'function' ? stream : Readable.from(stream)
this.#stream = stream
stream.on('readable', () => this.#resolve())
finished(stream, { writable: false }, error => {
this.#error = error
this.#ended = true
this.#resolve()
})
}
async read(size) {
if (size !== undefined) {
assert(size > 0)
}
do {
if (this.#ended) {
if (this.#error) {
throw this.#error
}
return null
}
const value = this.#stream.read(size)
if (value !== null) {
return value
}
await new Promise(this.#executor)
} while (true)
}
async readStrict(size) {
const chunk = await this.read(size)
if (chunk === null) {
throw new Error('stream has ended without data')
}
if (size !== undefined && chunk.length !== size) {
const error = new Error(`stream has ended with not enough data (actual: ${chunk.length}, expected: ${size})`)
Object.defineProperties(error, {
chunk: {
value: chunk,
},
})
throw error
}
return chunk
}
async skip(size) {
if (size === 0) {
return size
}
let toSkip = size
do {
if (this.#ended) {
if (this.#error) {
throw this.#error
}
return size - toSkip
}
const data = this.#stream.read()
if (data !== null) {
toSkip -= data === null ? 0 : data.length
if (toSkip > 0) {
// continue to read
} else {
// if more than wanted has been read, push back the rest
if (toSkip < 0) {
this.#stream.unshift(data.slice(toSkip))
}
return size
}
}
await new Promise(this.#executor)
} while (true)
}
async skipStrict(size) {
const bytesSkipped = await this.skip(size)
if (bytesSkipped !== size) {
const error = new Error(`stream has ended with not enough data (actual: ${bytesSkipped}, expected: ${size})`)
error.bytesSkipped = bytesSkipped
throw error
}
}
}
StreamReader.prototype[Symbol.asyncIterator] = async function* asyncIterator() {
let chunk
while ((chunk = await this.read()) !== null) {
yield chunk
}
}
module.exports = StreamReader

View File

@@ -0,0 +1,141 @@
'use strict'
const { describe, it } = require('test')
const assert = require('node:assert').strict
const { Readable } = require('stream')
const StreamReader = require('./index.js')
const makeStream = it => Readable.from(it, { objectMode: false })
makeStream.obj = Readable.from
const rejectionOf = promise =>
promise.then(
value => {
throw value
},
error => error
)
const makeErrorTests = method => {
it('rejects if the stream errors', async () => {
const error = new Error()
const stream = makeStream([])
const pError = rejectionOf(new StreamReader(stream)[method](10))
stream.destroy(error)
assert.strict(await pError, error)
})
it('rejects if the stream has already errored', async () => {
const error = new Error()
const stream = makeStream([])
await new Promise(resolve => {
stream.once('error', resolve).destroy(error)
})
assert.strict(await rejectionOf(new StreamReader(stream)[method](10)), error)
})
}
describe('read()', () => {
it('rejects if size is less than or equal to 0', async () => {
const error = await rejectionOf(new StreamReader(makeStream([])).read(0))
assert.strictEqual(error.code, 'ERR_ASSERTION')
})
it('returns null if stream is empty', async () => {
assert.strictEqual(await new StreamReader(makeStream([])).read(), null)
})
makeErrorTests('read')
it('returns null if the stream is already ended', async () => {
const reader = new StreamReader(makeStream([]))
await reader.read()
assert.strictEqual(await reader.read(), null)
})
describe('with binary stream', () => {
it('returns the first chunk of data', async () => {
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(), Buffer.from('foo'))
})
it('returns a chunk of the specified size (smaller than first)', async () => {
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(2), Buffer.from('fo'))
})
it('returns a chunk of the specified size (larger than first)', async () => {
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(4), Buffer.from('foob'))
})
it('returns less data if stream ends', async () => {
assert.deepEqual(await new StreamReader(makeStream(['foo', 'bar'])).read(10), Buffer.from('foobar'))
})
})
describe('with object stream', () => {
it('returns the first chunk of data verbatim', async () => {
const chunks = [{}, {}]
assert.strictEqual(await new StreamReader(makeStream.obj(chunks)).read(), chunks[0])
})
})
})
describe('readStrict()', function () {
it('throws if stream is empty', async () => {
const error = await rejectionOf(new StreamReader(makeStream([])).readStrict())
assert(error instanceof Error)
assert.strictEqual(error.message, 'stream has ended without data')
assert.strictEqual(error.chunk, undefined)
})
it('throws if stream ends with not enough data', async () => {
const error = await rejectionOf(new StreamReader(makeStream(['foo', 'bar'])).readStrict(10))
assert(error instanceof Error)
assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 6, expected: 10)')
assert.deepEqual(error.chunk, Buffer.from('foobar'))
})
})
describe('skip()', function () {
makeErrorTests('skip')
it('returns 0 if size is 0', async () => {
assert.strictEqual(await new StreamReader(makeStream(['foo'])).skip(0), 0)
})
it('returns 0 if the stream is already ended', async () => {
const reader = new StreamReader(makeStream([]))
await reader.read()
assert.strictEqual(await reader.skip(10), 0)
})
it('skips a number of bytes', async () => {
const reader = new StreamReader(makeStream('foo bar'))
assert.strictEqual(await reader.skip(4), 4)
assert.deepEqual(await reader.read(4), Buffer.from('bar'))
})
it('returns less size if stream ends', async () => {
assert.deepEqual(await new StreamReader(makeStream('foo bar')).skip(10), 7)
})
})
describe('skipStrict()', function () {
it('throws if stream ends with not enough data', async () => {
const error = await rejectionOf(new StreamReader(makeStream('foo bar')).skipStrict(10))
assert(error instanceof Error)
assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 7, expected: 10)')
assert.deepEqual(error.bytesSkipped, 7)
})
})

View File

@@ -0,0 +1,39 @@
{
"private": false,
"name": "@vates/stream-reader",
"description": "Efficiently reads and skips chunks of a given size in a stream",
"keywords": [
"async",
"chunk",
"data",
"node",
"promise",
"read",
"reader",
"skip",
"stream"
],
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/stream-reader",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/stream-reader",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.0.0",
"engines": {
"node": ">=10"
},
"scripts": {
"postversion": "npm publish --access public",
"test": "node--test"
},
"devDependencies": {
"test": "^3.3.0"
}
}

View File

@@ -49,6 +49,7 @@ const DEFAULT_VM_SETTINGS = {
timeout: 0,
useNbd: false,
unconditionalSnapshot: false,
validateVhdStreams: true,
vmTimeout: 0,
}

View File

@@ -6,11 +6,13 @@ const groupBy = require('lodash/groupBy.js')
const ignoreErrors = require('promise-toolbox/ignoreErrors')
const keyBy = require('lodash/keyBy.js')
const mapValues = require('lodash/mapValues.js')
const vhdStreamValidator = require('vhd-lib/vhdStreamValidator.js')
const { asyncMap } = require('@xen-orchestra/async-map')
const { createLogger } = require('@xen-orchestra/log')
const { decorateMethodsWith } = require('@vates/decorate-with')
const { defer } = require('golike-defer')
const { formatDateTime } = require('@xen-orchestra/xapi')
const { pipeline } = require('node:stream')
const { DeltaBackupWriter } = require('./writers/DeltaBackupWriter.js')
const { DeltaReplicationWriter } = require('./writers/DeltaReplicationWriter.js')
@@ -44,6 +46,8 @@ const forkDeltaExport = deltaExport =>
},
})
const noop = Function.prototype
class VmBackup {
constructor({
config,
@@ -251,6 +255,11 @@ class VmBackup {
Task.info('Transfer data using NBD')
}
const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
if (this._settings.validateVhdStreams) {
deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
}
deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
const timestamp = Date.now()

View File

@@ -34,7 +34,11 @@
<!--packages-start-->
- @vates/diff minor
- @vates/read-chunk patch
- @vates/stream-reader minor
- @xen-orchestra/backups minor
- vhd-lib minor
- xo-web patch
- xo-server minor
- xo-server-usage-report patch

View File

@@ -12,11 +12,13 @@
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"engines": {
"node": ">=14"
"node": ">=16"
},
"dependencies": {
"@vates/async-each": "^1.0.0",
"@vates/diff": "^0.0.0",
"@vates/read-chunk": "^1.1.0",
"@vates/stream-reader": "^0.0.0",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/fs": "^3.3.4",
"@xen-orchestra/log": "^0.6.0",

View File

@@ -0,0 +1,79 @@
'use strict'
const diff = require('@vates/diff')
const StreamReader = require('@vates/stream-reader')
const { FOOTER_SIZE, HEADER_SIZE } = require('./_constants.js')
const { unpackFooter, unpackHeader } = require('./Vhd/_utils.js')
// Async generator that can be in a pipeline to check the header and footers of a VHD stream.
//
// ```js
// await pipeline(createVhdStream(), vhdStreamValidator, createOutputStream())
// ```
module.exports = async function* vhdStreamValidator(source) {
let offset = 0
try {
const reader = new StreamReader(source)
const footer1 = await reader.readStrict(FOOTER_SIZE)
yield footer1
// unpack and validate footer
unpackFooter(footer1)
offset += footer1.length
const header = await reader.readStrict(HEADER_SIZE)
yield header
// unpack and validate header
unpackHeader(header)
offset += header.length
let last
while (true) {
const chunk = await reader.read(FOOTER_SIZE)
if (chunk === null) {
// the end has been reached, `last` contains the footer
break
}
yield chunk
offset += chunk.length
if (chunk.length < FOOTER_SIZE) {
// the end has been reached, the footer is part in `last` and part in `chunk`
last = Buffer.concat([last.slice(-(FOOTER_SIZE - chunk.length)), chunk])
break
}
last = chunk
}
// offset is the position at which the footer was read, not the position after
offset -= last.length
// do not use `assert.strictDeepEqual` as it does not provide a compact (for buffers) or complete (for strings) result
//
// - save only expected data and diff
// - save expected buffer as hex string for compacity and readability
// - save diff as an array of differences represented as hex strings
if (!last.equals(footer1)) {
const error = new Error('footer1 !== footer2')
error.diff = []
diff(footer1, last, (i, buf) => {
error.diff.push(i, buf.toString('hex'))
})
error.expected = footer1.toString('hex')
throw error
}
} catch (error) {
error.offset = offset
throw error
}
}