mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-09-10 22:14:48 -05:00
Customer imported large VM from VMWare (> 3 To), import switched to Qcow2 automatically but the process was stuck while building the qcow2 L1/L2 address tables, where the NbdDisk's hasBlock is called for each l2 table entry which iterated over the disk dataMap which contained ~113,528 ranges. This froze the process for more than 18 minutes after which it was either killed or died due to OOM error.
This fix improves the NbdDisk by:
Updating processDatamap to throw on overlapping ranges and merge touching extents.
Override default getBlockIndexesCount with optimized Nbd one.
Improve hasBlock with a built in cursor to improve performance.
Test file added to cover the new function and updated code.
128 lines
5.6 KiB
JavaScript
128 lines
5.6 KiB
JavaScript
import { describe, it } from 'node:test'
|
|
import assert from 'node:assert'
|
|
|
|
import { NbdDisk } from './NbdDisk.mjs'
|
|
|
|
const KiB = 1024
|
|
const MiB = 1024 * KiB
|
|
const TiB = 1024 * 1024 * MiB
|
|
|
|
const SOURCE_BLOCK_SIZE = 2 * MiB // VHD_BLOCK_SIZE, the block size used when importing from VMware
|
|
const CLUSTER_SIZE = 64 * KiB // qcow2 cluster size
|
|
|
|
// hasBlock() only needs the dataMap (passed through the constructor) and the
|
|
// block size, so we can exercise it without a live NBD connection.
|
|
describe('NbdDisk.hasBlock', () => {
|
|
it('reports allocated and unallocated blocks correctly', () => {
|
|
const dataMap = [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
|
|
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
|
|
]
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
|
|
assert.strictEqual(disk.hasBlock(0), true)
|
|
assert.strictEqual(disk.hasBlock(1), false)
|
|
assert.strictEqual(disk.hasBlock(4), false)
|
|
assert.strictEqual(disk.hasBlock(5), true)
|
|
assert.strictEqual(disk.hasBlock(6), false)
|
|
})
|
|
|
|
it('handles repeated and out-of-order queries with the resume cursor', () => {
|
|
const dataMap = [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
|
|
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
|
|
]
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
|
|
// repeated indexes (the qcow2 path queries each 2MB block ~32 times as it
|
|
// walks 64KB clusters) must return a stable answer
|
|
assert.strictEqual(disk.hasBlock(5), true)
|
|
assert.strictEqual(disk.hasBlock(5), true)
|
|
assert.strictEqual(disk.hasBlock(6), false)
|
|
assert.strictEqual(disk.hasBlock(6), false)
|
|
|
|
// querying backwards must still be correct (falls back to a full scan)
|
|
assert.strictEqual(disk.hasBlock(0), true)
|
|
assert.strictEqual(disk.hasBlock(5), true)
|
|
})
|
|
|
|
it('invalidates the cursor on a backward miss (no stale skip)', () => {
|
|
const dataMap = [
|
|
{ type: 0, offset: 3 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 3
|
|
{ type: 0, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 10
|
|
]
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
|
|
assert.strictEqual(disk.hasBlock(10), true) // forward hit: cursor parks on the high extent
|
|
assert.strictEqual(disk.hasBlock(1), false) // backward miss: must invalidate the cursor
|
|
assert.strictEqual(disk.hasBlock(3), true) // lower extent must not be skipped
|
|
})
|
|
|
|
it('throws on overlapping ranges (would break the resume cursor)', () => {
|
|
const dataMap = [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE },
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE }, // overlaps the previous range
|
|
]
|
|
assert.throws(() => new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap }), /overlapping ranges/)
|
|
})
|
|
|
|
it('merges touching ranges and drops zero-length / non type-0 entries', () => {
|
|
const dataMap = [
|
|
{ type: 0, offset: 2 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [2, 3)
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5)
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // [1, 2)
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 }, // [.5, 1), touches [0, .5)
|
|
{ type: 1, offset: 10 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // wrong type: ignored
|
|
{ type: 0, offset: 7 * SOURCE_BLOCK_SIZE, length: 0 }, // zero length: ignored
|
|
]
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
|
|
assert.strictEqual(disk.hasBlock(0), true)
|
|
assert.strictEqual(disk.hasBlock(1), true)
|
|
assert.strictEqual(disk.hasBlock(2), true)
|
|
assert.strictEqual(disk.hasBlock(3), false)
|
|
assert.strictEqual(disk.hasBlock(7), false) // zero-length range dropped
|
|
assert.strictEqual(disk.hasBlock(10), false) // non type-0 range dropped
|
|
})
|
|
})
|
|
|
|
describe('NbdDisk.getBlockIndexesCount', () => {
|
|
const cases = {
|
|
'aligned, disjoint blocks': [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE }, // block 0
|
|
{ type: 0, offset: 5 * SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE }, // block 5
|
|
],
|
|
'a range spanning several blocks': [
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: 3 * SOURCE_BLOCK_SIZE }, // blocks 0,1,2,3
|
|
],
|
|
'two disjoint extents sharing block 0 (needs dedup)': [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 }, // [0, .5) => block 0
|
|
{ type: 0, offset: 3 * (SOURCE_BLOCK_SIZE / 4), length: SOURCE_BLOCK_SIZE }, // [.75, 1.75) => blocks 0,1
|
|
],
|
|
'touching sub-block fragments': [
|
|
{ type: 0, offset: 0, length: SOURCE_BLOCK_SIZE / 2 },
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE / 2, length: SOURCE_BLOCK_SIZE / 2 },
|
|
{ type: 0, offset: SOURCE_BLOCK_SIZE, length: SOURCE_BLOCK_SIZE },
|
|
],
|
|
empty: [],
|
|
}
|
|
|
|
for (const [name, dataMap] of Object.entries(cases)) {
|
|
it(`matches getBlockIndexes().length: ${name}`, () => {
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
|
|
})
|
|
}
|
|
|
|
it('matches getBlockIndexes().length on the large sparse map', () => {
|
|
const virtualSize = Math.floor(3.5 * TiB)
|
|
const NB_RANGES = 113528
|
|
const dataMap = []
|
|
for (let i = 0; i < NB_RANGES; i++) {
|
|
dataMap.push({ type: 0, offset: Math.floor((i / NB_RANGES) * virtualSize), length: CLUSTER_SIZE })
|
|
}
|
|
const disk = new NbdDisk({}, SOURCE_BLOCK_SIZE, { dataMap })
|
|
assert.strictEqual(disk.getBlockIndexesCount(), disk.getBlockIndexes().length)
|
|
})
|
|
})
|