mirror of
https://github.com/vatesfr/xen-orchestra.git
synced 2026-09-10 22:14:48 -05:00
refactor(xo-server): remove hrp dependency (#10038)
replace http-request-plus external depedency by a a native wrapper around fetch
This commit is contained in:
@@ -7,18 +7,22 @@ import CSON from 'cson-parser'
|
||||
import fromCallback from 'promise-toolbox/fromCallback'
|
||||
import fs from 'fs'
|
||||
import getopts from 'getopts'
|
||||
import hrp from 'http-request-plus'
|
||||
import split2 from 'split2'
|
||||
import pumpify from 'pumpify'
|
||||
import { Agent } from 'undici'
|
||||
import { extname } from 'path'
|
||||
import { format, parse } from 'json-rpc-protocol'
|
||||
import { inspect } from 'util'
|
||||
import { load as loadConfig } from 'app-conf'
|
||||
import { pipeline } from 'stream'
|
||||
import { pipeline, Readable } from 'stream'
|
||||
import { readChunk } from '@vates/read-chunk'
|
||||
|
||||
const pkg = JSON.parse(fs.readFileSync(new URL('package.json', import.meta.url)))
|
||||
|
||||
// the proxy uses a self-signed certificate; timeouts disabled (0) because
|
||||
// requests can be long running (the default 300s would cut them off)
|
||||
const insecureAgent = new Agent({ connect: { rejectUnauthorized: false }, headersTimeout: 0, bodyTimeout: 0 })
|
||||
|
||||
const FORMATS = {
|
||||
__proto__: null,
|
||||
|
||||
@@ -94,10 +98,7 @@ ${pkg.name} v${pkg.version}`
|
||||
cookie: `authenticationToken=${token}`,
|
||||
},
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
|
||||
// Default 5s timeout (since Node 19) is problematic with long running requests
|
||||
timeout: 0,
|
||||
dispatcher: insecureAgent,
|
||||
}
|
||||
|
||||
const call = async ({ method, params }) => {
|
||||
@@ -105,25 +106,30 @@ ${pkg.name} v${pkg.version}`
|
||||
process.stderr.write(`\n${colors.bold(`--- call #${callPath.join('.')}`)} ---\n\n`)
|
||||
}
|
||||
|
||||
const response = await hrp(url, {
|
||||
const response = await fetch(url, {
|
||||
...baseRequest,
|
||||
|
||||
body: format.request(0, method, params),
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
const { stdout } = process
|
||||
const stream = Readable.fromWeb(response.body)
|
||||
|
||||
const responseType = contentType.parse(response).type
|
||||
const responseType = contentType.parse(response.headers.get('content-type'))?.type
|
||||
if (responseType === 'application/octet-stream') {
|
||||
if (stdout.isTTY) {
|
||||
throw new Error('binary data, pipe to a file!')
|
||||
}
|
||||
await fromCallback(pipeline, response, stdout)
|
||||
await fromCallback(pipeline, stream, stdout)
|
||||
return
|
||||
}
|
||||
|
||||
assert.strictEqual(responseType, 'application/json')
|
||||
const lines = pumpify.obj(response, split2())
|
||||
const lines = pumpify.obj(stream, split2())
|
||||
|
||||
const firstLine = await readChunk(lines)
|
||||
|
||||
|
||||
@@ -32,11 +32,11 @@
|
||||
"content-type": "^1.0.4",
|
||||
"cson-parser": "^4.0.7",
|
||||
"getopts": "^2.2.3",
|
||||
"http-request-plus": "^1.0.0",
|
||||
"json-rpc-protocol": "^0.13.1",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"pumpify": "^2.0.1",
|
||||
"split2": "^4.1.0"
|
||||
"split2": "^4.1.0",
|
||||
"undici": "^6.26.0"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
"form-data": "^4.0.0",
|
||||
"fs-extra": "^11.1.0",
|
||||
"get-stream": "^6.0.0",
|
||||
"http-request-plus": "^1.0.0",
|
||||
"human-format": "^1.0.0",
|
||||
"lodash": "^4.18.0",
|
||||
"pretty-ms": "^7.0.0",
|
||||
|
||||
@@ -9,7 +9,6 @@ import { createReadStream } from 'fs'
|
||||
import { stat } from 'fs-extra'
|
||||
import getStream from 'get-stream'
|
||||
import has from 'lodash/has'
|
||||
import hrp from 'http-request-plus'
|
||||
import humanFormat from 'human-format'
|
||||
import isObject from 'lodash/isObject'
|
||||
import getKeys from 'lodash/keys'
|
||||
@@ -229,18 +228,19 @@ export async function upload(args) {
|
||||
noop
|
||||
)
|
||||
formData.append('file', input, { filename: 'file', knownLength: length })
|
||||
try {
|
||||
const response = await hrp(url.toString(), { body: formData, headers: formData.getHeaders(), method: 'POST' })
|
||||
return await response.text()
|
||||
} catch (e) {
|
||||
console.log('ERROR', e)
|
||||
const { response } = e
|
||||
if (response !== undefined) {
|
||||
console.log('ERROR content', await response.text())
|
||||
}
|
||||
|
||||
throw e
|
||||
const response = await fetch(url.toString(), {
|
||||
body: formData,
|
||||
headers: formData.getHeaders(),
|
||||
method: 'POST',
|
||||
duplex: 'half', // required when sending a stream body
|
||||
})
|
||||
if (!response.ok) {
|
||||
const content = await response.text()
|
||||
console.log('ERROR', response.status, response.statusText)
|
||||
console.log('ERROR content', content)
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
return await response.text()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,10 +35,10 @@
|
||||
"@xen-orchestra/qcow2": "^1.3.1",
|
||||
"d3-time-format": "^4.1.0",
|
||||
"golike-defer": "^0.5.1",
|
||||
"http-request-plus": "^1.0.0",
|
||||
"json-rpc-protocol": "^0.13.2",
|
||||
"lodash": "^4.18.0",
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"undici": "^6.26.0",
|
||||
"vhd-lib": "^4.16.1",
|
||||
"xo-common": "^0.10.0"
|
||||
},
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import CancelToken from 'promise-toolbox/CancelToken'
|
||||
import groupBy from 'lodash/groupBy.js'
|
||||
import hrp from 'http-request-plus'
|
||||
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
||||
import pRetry from 'promise-toolbox/retry'
|
||||
import pickBy from 'lodash/pickBy.js'
|
||||
@@ -14,6 +13,7 @@ import { extract } from '@xen-orchestra/xapi/xoData.mjs'
|
||||
import { finished } from 'node:stream'
|
||||
import { incorrectState, forbiddenOperation } from 'xo-common/api-errors.js'
|
||||
import { JsonRpcError } from 'json-rpc-protocol'
|
||||
import { Agent } from 'undici'
|
||||
import { Ref } from 'xen-api'
|
||||
|
||||
import isDefaultTemplate from './isDefaultTemplate.mjs'
|
||||
@@ -21,6 +21,8 @@ import isVmRunning from './_isVmRunning.mjs'
|
||||
|
||||
const { warn, error } = createLogger('xo:xapi:vm')
|
||||
|
||||
const insecureAgent = new Agent({ connect: { rejectUnauthorized: false } })
|
||||
|
||||
const BIOS_STRINGS_KEYS = new Set([
|
||||
'baseboard-asset-tag',
|
||||
'baseboard-location-in-chassis',
|
||||
@@ -192,11 +194,16 @@ class Vm {
|
||||
}
|
||||
|
||||
try {
|
||||
await hrp(url, {
|
||||
const response = await fetch(url, {
|
||||
headers,
|
||||
rejectUnauthorized: false,
|
||||
timeout: this._syncHookTimeout ?? 60e3,
|
||||
dispatcher: insecureAgent,
|
||||
signal: AbortSignal.timeout(this._syncHookTimeout ?? 60e3),
|
||||
})
|
||||
// the response body is not used, free the socket
|
||||
await response.body?.cancel()
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
} catch (error) {
|
||||
warn('HTTP hook failed', { error, url, vm: uuid })
|
||||
}
|
||||
|
||||
@@ -40,11 +40,17 @@
|
||||
<!--packages-start-->
|
||||
|
||||
- @xen-orchestra/acl minor
|
||||
- @xen-orchestra/proxy-cli patch
|
||||
- @xen-orchestra/rest-api minor
|
||||
- @xen-orchestra/upload-ova patch
|
||||
- @xen-orchestra/web minor
|
||||
- @xen-orchestra/web-core minor
|
||||
- @xen-orchestra/xapi patch
|
||||
- xen-api minor
|
||||
- xo-cli patch
|
||||
- xo-common minor
|
||||
- xo-server patch
|
||||
- xo-server-ipmi-sensors minor
|
||||
- xo-server-netbox patch
|
||||
|
||||
|
||||
<!--packages-end-->
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import assert from 'assert'
|
||||
import dns from 'dns'
|
||||
import http from 'node:http'
|
||||
import https from 'node:https'
|
||||
import ms from 'ms'
|
||||
import httpRequest from 'http-request-plus'
|
||||
import map from 'lodash/map.js'
|
||||
import noop from 'lodash/noop.js'
|
||||
import Obfuscate from '@vates/obfuscate'
|
||||
@@ -16,6 +17,7 @@ import { jsonHash } from '@vates/json-hash'
|
||||
import { cancelable, defer, fromCallback, ignoreErrors, pDelay, pRetry, pTimeout } from 'promise-toolbox'
|
||||
import { limitConcurrency } from 'limit-concurrency-decorator'
|
||||
import { decorateClass } from '@vates/decorate-with'
|
||||
import { pipeline } from 'node:stream'
|
||||
import { ProxyAgent as HttpProxyAgent } from 'proxy-agent'
|
||||
|
||||
import getTaskResult from './_getTaskResult.mjs'
|
||||
@@ -31,6 +33,95 @@ import { noSuchObject } from 'xo-common/api-errors.js'
|
||||
const { debug } = createLogger('xen-api')
|
||||
|
||||
// ===================================================================
|
||||
// Minimal `node:http(s)` request helper used by `putResource`.
|
||||
// `fetch`/`undici` cannot be used here because `putResource` relies on a hack
|
||||
// (see below) where a huge `content-length` is announced and the connection is
|
||||
// cut once the real data has been sent
|
||||
// Mimics the subset of `http-request-plus` that `putResource` depends on:
|
||||
// resolves with the Node response on a 2xx status, otherwise rejects with an
|
||||
// error carrying the response as `error.response` (whose body is drained, so a
|
||||
// non-2xx response never leaks its socket). A stream body is never
|
||||
// replayed, so redirects are not followed for it (`putResource` probes for
|
||||
// redirections with an empty body beforehand).
|
||||
function nodeRequest(url, { body, headers, maxRedirects = 5, signal, timeout, ...opts }) {
|
||||
url = url instanceof URL ? url : new URL(url)
|
||||
debug('nodeRequest', url.href)
|
||||
const bodyIsStream = body != null && typeof body.pipe === 'function'
|
||||
|
||||
headers = { ...headers }
|
||||
if (body !== undefined && headers['content-length'] === undefined) {
|
||||
const length = bodyIsStream ? (body.headers?.['content-length'] ?? body.length) : Buffer.byteLength(body)
|
||||
if (length !== undefined) {
|
||||
headers['content-length'] = length
|
||||
}
|
||||
}
|
||||
|
||||
let redirectsLeft = bodyIsStream ? 0 : maxRedirects
|
||||
|
||||
const send = currentUrl =>
|
||||
new Promise((resolve, reject) => {
|
||||
const req = (currentUrl.protocol === 'https:' ? https : http).request(currentUrl, { ...opts, headers, signal })
|
||||
|
||||
// before the response is received, errors reject the promise; afterwards,
|
||||
// they are forwarded to the response so an abnormally closed connection
|
||||
// (the 1PiB hack in `putResource` cuts the socket once the data has been
|
||||
// sent) does not reject a settled promise, and `putResource`'s
|
||||
// post-processing can handle the `ERR_STREAM_PREMATURE_CLOSE` itself
|
||||
let sendError = reject
|
||||
const onError = error => sendError(error)
|
||||
req.on('error', onError)
|
||||
|
||||
if (timeout !== undefined) {
|
||||
req.setTimeout(timeout, () => {
|
||||
const error = new Error('HTTP connection has timed out')
|
||||
error.url = currentUrl.href
|
||||
req.destroy(error)
|
||||
})
|
||||
}
|
||||
|
||||
req.on('response', response => {
|
||||
const { statusCode } = response
|
||||
const { location } = response.headers
|
||||
const isRedirect = statusCode >= 300 && statusCode < 400
|
||||
|
||||
if (redirectsLeft > 0 && isRedirect && location !== undefined) {
|
||||
--redirectsLeft
|
||||
response.destroy()
|
||||
resolve(send(new URL(location, currentUrl)))
|
||||
return
|
||||
}
|
||||
|
||||
sendError = error => response.destroy(error)
|
||||
// a client `IncomingMessage` has no `.url`; expose the final URL so
|
||||
// callers (e.g. `putResource`) can attach it as error context
|
||||
response.url = currentUrl.href
|
||||
const isOk = statusCode >= 200 && statusCode < 300
|
||||
if (isOk) {
|
||||
resolve(response)
|
||||
} else {
|
||||
const error = new Error(`${statusCode} ${response.statusMessage}`)
|
||||
Object.defineProperty(error, 'response', { value: response })
|
||||
|
||||
response.destroy()
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
|
||||
if (bodyIsStream) {
|
||||
// let `pipeline` own the request errors while streaming
|
||||
req.off('error', onError)
|
||||
pipeline(body, req, error => {
|
||||
if (error != null) {
|
||||
sendError(error)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
req.end(body)
|
||||
}
|
||||
})
|
||||
|
||||
return send(url)
|
||||
}
|
||||
|
||||
// in seconds!
|
||||
const EVENT_TIMEOUT = 60
|
||||
@@ -562,7 +653,7 @@ export class Xapi extends EventEmitter {
|
||||
await this._setHostAddressInUrl(url, host)
|
||||
|
||||
const doRequest = (url, opts) =>
|
||||
httpRequest(url, {
|
||||
nodeRequest(url, {
|
||||
agent: this._httpAgent,
|
||||
body,
|
||||
headers,
|
||||
|
||||
@@ -41,7 +41,6 @@
|
||||
"@xen-orchestra/log": "^0.7.2",
|
||||
"bind-property-descriptor": "^2.0.0",
|
||||
"blocked": "^1.2.1",
|
||||
"http-request-plus": "^1.0.2",
|
||||
"jest-diff": "^29.0.3",
|
||||
"json-rpc-protocol": "^0.13.1",
|
||||
"limit-concurrency-decorator": "^0.6.0",
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { Agent } from 'undici'
|
||||
import { createReadStream, createWriteStream, readFileSync } from 'fs'
|
||||
import { PassThrough, pipeline } from 'stream'
|
||||
import { PassThrough, pipeline, Readable } from 'stream'
|
||||
import { stat } from 'fs/promises'
|
||||
import chalk from 'chalk'
|
||||
import forEach from 'lodash/forEach.js'
|
||||
import fromCallback from 'promise-toolbox/fromCallback'
|
||||
import getKeys from 'lodash/keys.js'
|
||||
import getopts from 'getopts'
|
||||
import hrp from 'http-request-plus'
|
||||
import identity from 'lodash/identity.js'
|
||||
import isObject from 'lodash/isObject.js'
|
||||
import micromatch from 'micromatch'
|
||||
@@ -624,10 +624,12 @@ async function call(args) {
|
||||
try {
|
||||
// FIXME: do not use private properties.
|
||||
const baseUrl = xo._url.replace(/^ws/, 'http')
|
||||
const httpOptions = {
|
||||
rejectUnauthorized: !(await getServerConfig()).allowUnauthorized,
|
||||
timeout: 0,
|
||||
}
|
||||
const rejectUnauthorized = !(await getServerConfig()).allowUnauthorized
|
||||
const dispatcher = new Agent({
|
||||
connect: { rejectUnauthorized },
|
||||
headersTimeout: 0,
|
||||
bodyTimeout: 0,
|
||||
})
|
||||
|
||||
const result = await xo.call(method, params)
|
||||
let keys, key, url
|
||||
@@ -638,9 +640,18 @@ async function call(args) {
|
||||
ensurePathParam(method, file)
|
||||
url = new URL(result[key], baseUrl)
|
||||
const output = createOutputStream(file)
|
||||
const response = await hrp(url, httpOptions)
|
||||
const response = await fetch(url, { dispatcher })
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
return fromCallback(pipeline, response, streamStatsPrinter(response.headers['content-length']), output)
|
||||
return fromCallback(
|
||||
pipeline,
|
||||
Readable.fromWeb(response.body),
|
||||
streamStatsPrinter(response.headers.get('content-length')),
|
||||
output
|
||||
)
|
||||
}
|
||||
|
||||
if (key === '$sendTo') {
|
||||
@@ -650,14 +661,17 @@ async function call(args) {
|
||||
const length = file === '-' ? undefined : (await stat(file)).size
|
||||
const input = pipeline(file === '-' ? process.stdin : createReadStream(file), streamStatsPrinter(length), noop)
|
||||
|
||||
const response = await hrp(url, {
|
||||
...httpOptions,
|
||||
const response = await fetch(url, {
|
||||
dispatcher,
|
||||
body: input,
|
||||
headers: length && {
|
||||
'content-length': length,
|
||||
},
|
||||
duplex: 'half',
|
||||
headers: length !== undefined ? { 'content-length': length } : {},
|
||||
method: 'POST',
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
return response.text()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,6 @@
|
||||
"content-type": "^1.0.5",
|
||||
"fs-extra": "^11.1.0",
|
||||
"getopts": "^2.3.0",
|
||||
"http-request-plus": "^1.0.0",
|
||||
"human-format": "^1.0.0",
|
||||
"lodash": "^4.18.0",
|
||||
"micromatch": "^4.0.2",
|
||||
@@ -43,6 +42,7 @@
|
||||
"promise-toolbox": "^0.21.0",
|
||||
"pw": "^0.0.4",
|
||||
"split2": "^4.2.0",
|
||||
"undici": "^6.26.0",
|
||||
"xdg-basedir": "^5.1.0",
|
||||
"xo-lib": "^0.11.2"
|
||||
},
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import { basename, join } from 'node:path'
|
||||
import { createReadStream, createWriteStream } from 'node:fs'
|
||||
import { normalize } from 'node:path/posix'
|
||||
import { Agent } from 'undici'
|
||||
import { parse as parseContentType } from 'content-type'
|
||||
import { pipeline } from 'node:stream'
|
||||
import { pipeline, Readable } from 'node:stream'
|
||||
import { pipeline as pPipeline } from 'node:stream/promises'
|
||||
import { readChunk } from '@vates/read-chunk'
|
||||
import { stat } from 'node:fs/promises'
|
||||
import getopts from 'getopts'
|
||||
import hrp from 'http-request-plus'
|
||||
import merge from 'lodash/merge.js'
|
||||
import set from 'lodash/set.js'
|
||||
import split2 from 'split2'
|
||||
@@ -74,10 +74,14 @@ const COMMANDS = {
|
||||
output === '-'
|
||||
? process.stdout
|
||||
: createWriteStream(output.endsWith('/') ? join(output, basename(path)) : output, { flags: 'wx' })
|
||||
return pPipeline(response, streamStatsPrinter(response.headers['content-length']), outputStream)
|
||||
return pPipeline(
|
||||
Readable.fromWeb(response.body),
|
||||
streamStatsPrinter(response.headers.get('content-length')),
|
||||
outputStream
|
||||
)
|
||||
}
|
||||
|
||||
const { type } = parseContentType(response)
|
||||
const { type } = parseContentType(response.headers.get('content-type'))
|
||||
if (type === 'application/json') {
|
||||
const result = await response.json()
|
||||
|
||||
@@ -94,7 +98,7 @@ const COMMANDS = {
|
||||
|
||||
return this.json ? JSON.stringify(result, null, 2) : result
|
||||
} else if (type === 'application/x-ndjson') {
|
||||
const lines = pipeline(response, split2(), noop)
|
||||
const lines = pipeline(Readable.fromWeb(response.body), split2(), noop)
|
||||
let line
|
||||
while ((line = await readChunk(lines)) !== null) {
|
||||
const data = JSON.parse(line)
|
||||
@@ -162,13 +166,14 @@ export async function rest(args) {
|
||||
// FIXME: extract server parsing in dedicated module/function
|
||||
const baseUrl = new Xo({ url: server })._url.replace(/^ws/, 'http')
|
||||
|
||||
const baseOpts = {
|
||||
headers: {
|
||||
cookie: 'authenticationToken=' + token,
|
||||
},
|
||||
rejectUnauthorized: !allowUnauthorized,
|
||||
timeout: 0,
|
||||
const baseHeaders = {
|
||||
cookie: 'authenticationToken=' + token,
|
||||
}
|
||||
const dispatcher = new Agent({
|
||||
connect: { rejectUnauthorized: !allowUnauthorized },
|
||||
headersTimeout: 0,
|
||||
bodyTimeout: 0,
|
||||
})
|
||||
|
||||
if (command === undefined || !(command in COMMANDS)) {
|
||||
return console.log('Available commands: ', Object.keys(COMMANDS).sort().join(', '))
|
||||
@@ -195,17 +200,20 @@ export async function rest(args) {
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await hrp(url, merge({}, baseOpts, opts))
|
||||
} catch (error) {
|
||||
const { response } = error
|
||||
if (response === undefined) {
|
||||
throw error
|
||||
}
|
||||
|
||||
console.error(response.statusCode, response.statusMessage)
|
||||
const merged = merge({ headers: { ...baseHeaders } }, opts)
|
||||
const { body } = merged
|
||||
const isBodyStream = body !== undefined && typeof body.pipe === 'function'
|
||||
const duplex = isBodyStream ? { duplex: 'half' } : {}
|
||||
const response = await fetch(url, {
|
||||
...merged,
|
||||
dispatcher,
|
||||
...duplex,
|
||||
})
|
||||
if (!response.ok) {
|
||||
console.error(response.status, response.statusText)
|
||||
throw await response.text()
|
||||
}
|
||||
return response
|
||||
},
|
||||
json,
|
||||
},
|
||||
|
||||
@@ -165,35 +165,32 @@ class Netbox {
|
||||
}
|
||||
|
||||
const httpRequest = async () => {
|
||||
let response
|
||||
let resBody = 'Netbox error could not be retrieved'
|
||||
try {
|
||||
response = await this.#xo.httpRequest(url, options)
|
||||
if (Math.floor(response.statusCode / 100) === 2) {
|
||||
resBody = await response.text()
|
||||
if (resBody.length > 0) {
|
||||
return JSON.parse(resBody)
|
||||
}
|
||||
return
|
||||
const response = await this.#xo.httpRequest(url, options)
|
||||
const resBody = await response.text()
|
||||
if (resBody.length > 0) {
|
||||
return JSON.parse(resBody)
|
||||
}
|
||||
const error = new Error(`${response.statusCode} ${response.statusMessage}`)
|
||||
try {
|
||||
resBody = await response.text()
|
||||
error.netboxError = JSON.parse(resBody)
|
||||
} catch (err) {
|
||||
log.error(err)
|
||||
// If the error couldn't be parsed, expose the response's raw body
|
||||
error.netboxError = resBody
|
||||
}
|
||||
throw error
|
||||
} catch (error) {
|
||||
// root error won't have a body , most likely an error code line ECONRESET
|
||||
error.method = method
|
||||
error.requestBody = dataDebug
|
||||
error.response = response
|
||||
|
||||
// On a non-2xx response, `httpRequest` throws with the (undrained)
|
||||
// response attached as `error.response` consume its body to expose the Netbox error. A root error (e.g.
|
||||
// ECONNRESET) has no response and is rethrown as-is.
|
||||
const { response } = error
|
||||
if (response !== undefined) {
|
||||
let resBody = 'Netbox error could not be retrieved'
|
||||
try {
|
||||
resBody = await response.text()
|
||||
error.netboxError = JSON.parse(resBody)
|
||||
} catch (err) {
|
||||
log.error(err)
|
||||
// If the error couldn't be parsed, expose the response's raw body
|
||||
error.netboxError = resBody
|
||||
}
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
response?.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +246,7 @@ class Netbox {
|
||||
try {
|
||||
this.#netboxVersion = semver.coerce((await this.#request('/status/'))['netbox-version']).version
|
||||
} catch (err) {
|
||||
if (err?.response?.statusCode === 404) {
|
||||
if (err?.response?.status === 404) {
|
||||
// Endpoint not supported on versions prior to v2.10
|
||||
// Best effort to support earlier versions without knowing the version explicitly
|
||||
return
|
||||
|
||||
@@ -94,7 +94,6 @@
|
||||
"helmet": "^3.9.0",
|
||||
"highland": "^2.11.1",
|
||||
"http-proxy": "^1.16.2",
|
||||
"http-request-plus": "^1.0.3",
|
||||
"@vates/http-server-plus": "^2.0.2",
|
||||
"human-format": "^1.0.0",
|
||||
"iterable-backoff": "^0.1.0",
|
||||
@@ -140,6 +139,7 @@
|
||||
"syslog-client": "^1.1.1",
|
||||
"tar-stream": "^3.1.6",
|
||||
"tmp": "^0.2.1",
|
||||
"undici": "^6.26.0",
|
||||
"unzipper": "^0.10.5",
|
||||
"uuid": "^9.0.0",
|
||||
"value-matcher": "^0.2.0",
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import * as multiparty from 'multiparty'
|
||||
import assert from 'assert'
|
||||
import hrp from 'http-request-plus'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { defer } from 'golike-defer'
|
||||
import { format, JsonRpcError } from 'json-rpc-peer'
|
||||
import { getStreamAsBuffer } from 'get-stream'
|
||||
import { invalidParameters, noSuchObject } from 'xo-common/api-errors.js'
|
||||
import { pipeline } from 'stream'
|
||||
import { pipeline, Readable } from 'stream'
|
||||
import { peekFooterFromVhdStream } from 'vhd-lib'
|
||||
import { vmdkToVhd } from 'xo-vmdk-to-vhd'
|
||||
|
||||
@@ -305,9 +304,14 @@ async function importDisk({ sr, type, name, description, url, vmdkData }) {
|
||||
throw invalidParameters('URL import is only compatible with VHD and raw formats')
|
||||
}
|
||||
|
||||
const stream = await hrp(url)
|
||||
const length = stream.headers['content-length']
|
||||
if (length !== undefined) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
const stream = Readable.fromWeb(response.body)
|
||||
const length = response.headers.get('content-length')
|
||||
if (length !== null) {
|
||||
stream.length = length
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { createLogger } from '@xen-orchestra/log'
|
||||
import assert from 'assert'
|
||||
import { format } from 'json-rpc-peer'
|
||||
import { X509Certificate } from 'node:crypto'
|
||||
import { pipeline } from 'node:stream'
|
||||
import { pipeline, Readable } from 'node:stream'
|
||||
import semver from 'semver'
|
||||
import { incorrectState, invalidParameters } from 'xo-common/api-errors.js'
|
||||
|
||||
@@ -742,7 +742,7 @@ async function handleGetSystemStatus(req, res, { xapi, host }) {
|
||||
|
||||
// Download and stream to client
|
||||
const response = await this.httpRequest(url, opts)
|
||||
return fromCallback(pipeline, response, res)
|
||||
return fromCallback(pipeline, Readable.fromWeb(response.body), res)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,7 +2,7 @@ import TTLCache from '@isaacs/ttlcache'
|
||||
import { asyncMap } from '@xen-orchestra/async-map'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { format } from 'json-rpc-peer'
|
||||
import { pipeline } from 'node:stream'
|
||||
import { pipeline, Readable } from 'node:stream'
|
||||
import tarStream from 'tar-stream'
|
||||
import { Ref } from 'xen-api'
|
||||
import { incorrectState, invalidParameters } from 'xo-common/api-errors.js'
|
||||
@@ -519,9 +519,8 @@ async function handleGetSystemStatuses(_req, res, { xapi, pool }) {
|
||||
const filename = `${host.name_label}-system-status.tar.bz2`
|
||||
|
||||
// Get the size from Content-Length header if available
|
||||
const size = response.headers['content-length']
|
||||
? Number.parseInt(response.headers['content-length'], 10)
|
||||
: undefined
|
||||
const contentLength = response.headers.get('content-length')
|
||||
const size = contentLength ? Number.parseInt(contentLength, 10) : undefined
|
||||
|
||||
if (size === undefined) {
|
||||
throw new Error(`Missing Content-Length header for host ${host.name_label} system status download`)
|
||||
@@ -539,7 +538,7 @@ async function handleGetSystemStatuses(_req, res, { xapi, pool }) {
|
||||
}
|
||||
})
|
||||
|
||||
await fromCallback(pipeline, response, entry)
|
||||
await fromCallback(pipeline, Readable.fromWeb(response.body), entry)
|
||||
}
|
||||
|
||||
// Finalize archive after all downloads complete
|
||||
|
||||
@@ -6,7 +6,6 @@ import { asyncEach } from '@vates/async-each'
|
||||
import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
|
||||
import { Task } from '@xen-orchestra/mixins/Tasks.mjs'
|
||||
import concat from 'lodash/concat.js'
|
||||
import hrp from 'http-request-plus'
|
||||
import mapKeys from 'lodash/mapKeys.js'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { defer } from 'golike-defer'
|
||||
@@ -15,6 +14,8 @@ import { FAIL_ON_QUEUE } from 'limit-concurrency-decorator'
|
||||
import { getStreamAsBuffer } from 'get-stream'
|
||||
import { ignoreErrors, timeout } from 'promise-toolbox'
|
||||
import { invalidParameters, noSuchObject, unauthorized } from 'xo-common/api-errors.js'
|
||||
import { Agent } from 'undici'
|
||||
import { Readable } from 'node:stream'
|
||||
import { Ref } from 'xen-api'
|
||||
|
||||
import { forEach, map, mapFilter, noop, parseSize, safeDateFormat } from '../utils.mjs'
|
||||
@@ -1367,8 +1368,19 @@ async function import_({ data, sr, type = 'xva', url }) {
|
||||
}
|
||||
|
||||
const timeout = this.config.getOptionalDuration('jsonrpc-api.xvaImportFromUrlTimeout') ?? 6e3
|
||||
const ref = await xapi.VM_import(await hrp(url, { timeout }), sr._xapiRef)
|
||||
return xapi.call('VM.get_uuid', ref)
|
||||
// `timeout` is an inactivity timeout: undici's headers/body timeouts match this semantic
|
||||
const dispatcher = new Agent({ headersTimeout: timeout, bodyTimeout: timeout })
|
||||
try {
|
||||
const response = await fetch(url, { dispatcher })
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
const ref = await xapi.VM_import(Readable.fromWeb(response.body), sr._xapiRef)
|
||||
return await xapi.call('VM.get_uuid', ref)
|
||||
} finally {
|
||||
await dispatcher.close()
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -6,6 +6,7 @@ import semver from 'semver'
|
||||
import some from 'lodash/some.js'
|
||||
import unzip from 'unzipper'
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
import { Readable } from 'node:stream'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { decorateObject } from '@vates/decorate-with'
|
||||
import { defer as deferrable } from 'golike-defer'
|
||||
@@ -79,7 +80,7 @@ const methods = {
|
||||
async _getXenUpdates() {
|
||||
const response = await this.xo.httpRequest('https://updates.ops.xenserver.com/xenserver/updates.xml')
|
||||
|
||||
const data = parseXml(await response.buffer()).patchdata
|
||||
const data = parseXml(Buffer.from(await response.arrayBuffer())).patchdata
|
||||
|
||||
const patches = { __proto__: null }
|
||||
forEach(data.patches.patch, patch => {
|
||||
@@ -463,7 +464,8 @@ const methods = {
|
||||
}
|
||||
|
||||
const { username, apikey } = xsCredentials
|
||||
let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
|
||||
const response = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
|
||||
let stream = Readable.fromWeb(response.body)
|
||||
stream = await new Promise((resolve, reject) => {
|
||||
const PATCH_RE = /\.xsupdate$/
|
||||
stream
|
||||
@@ -497,7 +499,8 @@ const methods = {
|
||||
}
|
||||
|
||||
const { username, apikey } = xsCredentials
|
||||
let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
|
||||
const response = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
|
||||
let stream = Readable.fromWeb(response.body)
|
||||
stream = await new Promise((resolve, reject) => {
|
||||
stream
|
||||
.pipe(unzip.Parse())
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
import hrp from 'http-request-plus'
|
||||
import { EnvHttpProxyAgent } from 'undici'
|
||||
import { ProxyAgent } from 'proxy-agent'
|
||||
|
||||
const { env } = process
|
||||
|
||||
export default class Http {
|
||||
// automatically fetches the proxy to use for the requested URL from the environment
|
||||
//
|
||||
// still exposed as `httpAgent` and used as a Node HTTP agent by other parts of the app
|
||||
_agent = new ProxyAgent()
|
||||
|
||||
// undici dispatchers for `fetch`, memoized by `${rejectUnauthorized}:${timeout}`
|
||||
//
|
||||
// `EnvHttpProxyAgent` reads the proxy configuration from the environment, kept
|
||||
// up to date by `setHttpProxy`
|
||||
_dispatchers = new Map()
|
||||
|
||||
// whether XO has a proxy set from its own config/environment
|
||||
get hasOwnHttpProxy() {
|
||||
return this._hasOwnHttpProxy
|
||||
@@ -26,13 +34,54 @@ export default class Http {
|
||||
})
|
||||
}
|
||||
|
||||
async httpRequest(url, opts) {
|
||||
try {
|
||||
return await hrp(url, { ...opts, agent: this._agent })
|
||||
} catch (error) {
|
||||
error.response?.destroy()
|
||||
_getDispatcher(rejectUnauthorized, timeout) {
|
||||
const key = `${rejectUnauthorized}:${timeout}`
|
||||
let dispatcher = this._dispatchers.get(key)
|
||||
if (dispatcher === undefined) {
|
||||
dispatcher = new EnvHttpProxyAgent({
|
||||
connect: { rejectUnauthorized },
|
||||
bodyTimeout: timeout,
|
||||
headersTimeout: timeout,
|
||||
})
|
||||
this._dispatchers.set(key, dispatcher)
|
||||
}
|
||||
return dispatcher
|
||||
}
|
||||
|
||||
// by default the failed response body is drained (freeing the socket) and its
|
||||
// content exposed as `error.data`; pass `bypassStatusCheck: true` to instead
|
||||
// take ownership of `error.response` and consume its body yourself
|
||||
async httpRequest(
|
||||
url,
|
||||
{ auth, body, bypassStatusCheck = false, headers, rejectUnauthorized = true, timeout = 0, ...opts } = {}
|
||||
) {
|
||||
const finalHeaders = { ...headers }
|
||||
if (auth !== undefined) {
|
||||
finalHeaders.authorization = 'Basic ' + Buffer.from(auth).toString('base64')
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
...opts,
|
||||
body,
|
||||
headers: finalHeaders,
|
||||
dispatcher: this._getDispatcher(rejectUnauthorized, timeout),
|
||||
|
||||
...(body !== undefined && typeof body.pipe === 'function' ? { duplex: 'half' } : {}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const error = new Error(`${response.status} ${response.statusText}`)
|
||||
error.response = response
|
||||
|
||||
if (!bypassStatusCheck) {
|
||||
// drain the response body to free the socket
|
||||
error.data = (await response.text().catch(() => '')).substring(0, 1024)
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
// Inject the proxy into the environment, it will be automatically used by `_agent` and by most libs (e.g `axios`)
|
||||
@@ -54,5 +103,12 @@ export default class Http {
|
||||
env.no_proxy = env.NO_PROXY = noProxy
|
||||
}
|
||||
}
|
||||
|
||||
// the proxy configuration changed: drop cached dispatchers so they are rebuilt
|
||||
// from the new environment
|
||||
for (const dispatcher of this._dispatchers.values()) {
|
||||
dispatcher.close().catch(() => {})
|
||||
}
|
||||
this._dispatchers.clear()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import assert from 'assert'
|
||||
import contentType from 'content-type'
|
||||
import cookie from 'cookie'
|
||||
import hrp from 'http-request-plus'
|
||||
import isEmpty from 'lodash/isEmpty.js'
|
||||
import omit from 'lodash/omit.js'
|
||||
import parseSetCookie from 'set-cookie-parser'
|
||||
import pumpify from 'pumpify'
|
||||
import some from 'lodash/some.js'
|
||||
import split2 from 'split2'
|
||||
import { Agent } from 'undici'
|
||||
import { compileTemplate } from '@xen-orchestra/template'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { decorateWith } from '@vates/decorate-with'
|
||||
@@ -16,6 +16,7 @@ import { format, parse } from 'json-rpc-peer'
|
||||
import { incorrectState, invalidParameters, noSuchObject } from 'xo-common/api-errors.js'
|
||||
import { parseDuration } from '@vates/parse-duration'
|
||||
import { readChunk, readChunkStrict } from '@vates/read-chunk'
|
||||
import { Readable } from 'node:stream'
|
||||
import { Ref } from 'xen-api'
|
||||
import { synchronized } from 'decorator-synchronized'
|
||||
import { timeout } from 'promise-toolbox'
|
||||
@@ -75,6 +76,8 @@ async function populateProxy(proxy) {
|
||||
export default class Proxy {
|
||||
constructor(app) {
|
||||
this._app = app
|
||||
|
||||
this._agents = new Map()
|
||||
const rules = {
|
||||
'{date}': (date = new Date()) => date.toISOString(),
|
||||
}
|
||||
@@ -100,6 +103,19 @@ export default class Proxy {
|
||||
})
|
||||
}
|
||||
|
||||
_getAgent(timeout) {
|
||||
let agent = this._agents.get(timeout)
|
||||
if (agent === undefined) {
|
||||
agent = new Agent({
|
||||
connect: { rejectUnauthorized: false },
|
||||
headersTimeout: timeout,
|
||||
bodyTimeout: timeout,
|
||||
})
|
||||
this._agents.set(timeout, agent)
|
||||
}
|
||||
return agent
|
||||
}
|
||||
|
||||
async _getChannel() {
|
||||
const PLACEHOLDER = '{xoChannel}'
|
||||
let channel = this._app.config.get('xo-proxy.channel')
|
||||
@@ -466,8 +482,7 @@ export default class Proxy {
|
||||
Cookie: cookie.serialize('authenticationToken', proxy.authenticationToken),
|
||||
},
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
timeout,
|
||||
dispatcher: this._getAgent(timeout),
|
||||
}
|
||||
|
||||
if (proxy.address !== undefined) {
|
||||
@@ -485,27 +500,33 @@ export default class Proxy {
|
||||
url.hostname = address.includes(':') ? `[${address}]` : address
|
||||
}
|
||||
|
||||
const response = await hrp(url, request)
|
||||
const response = await fetch(url, request)
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel() // free the socket
|
||||
throw new Error(`${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
const authenticationToken = parseSetCookie(response, {
|
||||
const authenticationToken = parseSetCookie(response.headers.getSetCookie(), {
|
||||
map: true,
|
||||
}).authenticationToken?.value
|
||||
if (authenticationToken !== undefined) {
|
||||
await this.updateProxy(id, { authenticationToken })
|
||||
}
|
||||
|
||||
const responseType = contentType.parse(response).type
|
||||
const stream = Readable.fromWeb(response.body)
|
||||
|
||||
const responseType = contentType.parse(response.headers.get('content-type'))?.type
|
||||
if (responseType === 'application/octet-stream') {
|
||||
if (assertType !== 'stream') {
|
||||
response.destroy()
|
||||
stream.destroy()
|
||||
throw new Error(`expect the result to be ${assertType}`)
|
||||
}
|
||||
return response
|
||||
return stream
|
||||
}
|
||||
|
||||
assert.strictEqual(responseType, 'application/json')
|
||||
|
||||
const lines = pumpify.obj(response, split2(JSON.parse))
|
||||
const lines = pumpify.obj(stream, split2(JSON.parse))
|
||||
const firstLine = await readChunk(lines)
|
||||
|
||||
const result = parse.result(firstLine)
|
||||
|
||||
20
yarn.lock
20
yarn.lock
@@ -4690,14 +4690,6 @@
|
||||
resolved "https://registry.yarnpkg.com/@vueuse/shared/-/shared-14.3.0.tgz#a3e7e6391f9ed7f363cbb28c32c4a278efaacbd0"
|
||||
integrity sha512-bZpge9eSXwa4ToSiqJ7j6KRwhAsneMFoSz3LMWKQDkqimm3D/tbFlrklrs/IOqC8tEcYmXQZJ6N0UrjhBirVCg==
|
||||
|
||||
"@xen-orchestra/log@^0.6.0":
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/@xen-orchestra/log/-/log-0.6.0.tgz#b7341818dbdc3a56facf815e7124867bf39ea89d"
|
||||
integrity sha512-IA1J5LRfFZdb/TaPLG4oeZl1kApJW5BUWvmsr33eFXMX/KWS/FQt1Lzsl2OpH4AwNw8SpUCCl2+3wC5fBDtLlw==
|
||||
dependencies:
|
||||
lodash "^4.17.4"
|
||||
promise-toolbox "^0.21.0"
|
||||
|
||||
"@xmldom/is-dom-node@^1.0.1":
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/@xmldom/is-dom-node/-/is-dom-node-1.0.1.tgz#83b9f3e1260fb008061c6fa787b93a00f9be0629"
|
||||
@@ -11144,13 +11136,6 @@ http-proxy@^1.16.2:
|
||||
follow-redirects "^1.0.0"
|
||||
requires-port "^1.0.0"
|
||||
|
||||
http-request-plus@^1.0.0, http-request-plus@^1.0.2, http-request-plus@^1.0.3:
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-1.0.3.tgz#34392950412b95056e05192fe7da15ec2c547c3b"
|
||||
integrity sha512-566RDaiBFQ7xfEE01Kw+qjNSBD+a893gJArx/PtQg9Bex5TxGedav7nOOBJzB2/+ZbZ5mMt5aHdiju1sJB2qBw==
|
||||
dependencies:
|
||||
"@xen-orchestra/log" "^0.6.0"
|
||||
|
||||
http-server-plus@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/http-server-plus/-/http-server-plus-1.0.0.tgz#cb7adc1ca3e679e8728286a6c9b5b1bd012ccbfd"
|
||||
@@ -19225,6 +19210,11 @@ undici@^6.2.1, undici@^6.5.0:
|
||||
resolved "https://registry.yarnpkg.com/undici/-/undici-6.26.0.tgz#333a35b7f519c48d2dc6aeb38e4e91d9274e0652"
|
||||
integrity sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A==
|
||||
|
||||
undici@^6.26.0:
|
||||
version "6.27.0"
|
||||
resolved "https://registry.yarnpkg.com/undici/-/undici-6.27.0.tgz#41f9e48f7c5a40d27376caaead8c9a9fc7bca9c4"
|
||||
integrity sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==
|
||||
|
||||
undici@^7.8.0:
|
||||
version "7.27.2"
|
||||
resolved "https://registry.yarnpkg.com/undici/-/undici-7.27.2.tgz#f8fae968ee68377cfc61713d9cd152773716804f"
|
||||
|
||||
Reference in New Issue
Block a user