refactor(xo-server): remove hrp dependency (#10038)

replace http-request-plus external depedency by a a native wrapper around fetch
This commit is contained in:
Grandalf
2026-08-07 11:58:20 +02:00
committed by GitHub
parent a0193085f5
commit 3293e69cc2
22 changed files with 353 additions and 141 deletions

View File

@@ -7,18 +7,22 @@ import CSON from 'cson-parser'
import fromCallback from 'promise-toolbox/fromCallback'
import fs from 'fs'
import getopts from 'getopts'
import hrp from 'http-request-plus'
import split2 from 'split2'
import pumpify from 'pumpify'
import { Agent } from 'undici'
import { extname } from 'path'
import { format, parse } from 'json-rpc-protocol'
import { inspect } from 'util'
import { load as loadConfig } from 'app-conf'
import { pipeline } from 'stream'
import { pipeline, Readable } from 'stream'
import { readChunk } from '@vates/read-chunk'
const pkg = JSON.parse(fs.readFileSync(new URL('package.json', import.meta.url)))
// the proxy uses a self-signed certificate; timeouts disabled (0) because
// requests can be long running (the default 300s would cut them off)
const insecureAgent = new Agent({ connect: { rejectUnauthorized: false }, headersTimeout: 0, bodyTimeout: 0 })
const FORMATS = {
__proto__: null,
@@ -94,10 +98,7 @@ ${pkg.name} v${pkg.version}`
cookie: `authenticationToken=${token}`,
},
method: 'POST',
rejectUnauthorized: false,
// Default 5s timeout (since Node 19) is problematic with long running requests
timeout: 0,
dispatcher: insecureAgent,
}
const call = async ({ method, params }) => {
@@ -105,25 +106,30 @@ ${pkg.name} v${pkg.version}`
process.stderr.write(`\n${colors.bold(`--- call #${callPath.join('.')}`)} ---\n\n`)
}
const response = await hrp(url, {
const response = await fetch(url, {
...baseRequest,
body: format.request(0, method, params),
})
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
const { stdout } = process
const stream = Readable.fromWeb(response.body)
const responseType = contentType.parse(response).type
const responseType = contentType.parse(response.headers.get('content-type'))?.type
if (responseType === 'application/octet-stream') {
if (stdout.isTTY) {
throw new Error('binary data, pipe to a file!')
}
await fromCallback(pipeline, response, stdout)
await fromCallback(pipeline, stream, stdout)
return
}
assert.strictEqual(responseType, 'application/json')
const lines = pumpify.obj(response, split2())
const lines = pumpify.obj(stream, split2())
const firstLine = await readChunk(lines)

View File

@@ -32,11 +32,11 @@
"content-type": "^1.0.4",
"cson-parser": "^4.0.7",
"getopts": "^2.2.3",
"http-request-plus": "^1.0.0",
"json-rpc-protocol": "^0.13.1",
"promise-toolbox": "^0.21.0",
"pumpify": "^2.0.1",
"split2": "^4.1.0"
"split2": "^4.1.0",
"undici": "^6.26.0"
},
"scripts": {
"postversion": "npm publish --access public"

View File

@@ -35,7 +35,6 @@
"form-data": "^4.0.0",
"fs-extra": "^11.1.0",
"get-stream": "^6.0.0",
"http-request-plus": "^1.0.0",
"human-format": "^1.0.0",
"lodash": "^4.18.0",
"pretty-ms": "^7.0.0",

View File

@@ -9,7 +9,6 @@ import { createReadStream } from 'fs'
import { stat } from 'fs-extra'
import getStream from 'get-stream'
import has from 'lodash/has'
import hrp from 'http-request-plus'
import humanFormat from 'human-format'
import isObject from 'lodash/isObject'
import getKeys from 'lodash/keys'
@@ -229,18 +228,19 @@ export async function upload(args) {
noop
)
formData.append('file', input, { filename: 'file', knownLength: length })
try {
const response = await hrp(url.toString(), { body: formData, headers: formData.getHeaders(), method: 'POST' })
return await response.text()
} catch (e) {
console.log('ERROR', e)
const { response } = e
if (response !== undefined) {
console.log('ERROR content', await response.text())
}
throw e
const response = await fetch(url.toString(), {
body: formData,
headers: formData.getHeaders(),
method: 'POST',
duplex: 'half', // required when sending a stream body
})
if (!response.ok) {
const content = await response.text()
console.log('ERROR', response.status, response.statusText)
console.log('ERROR content', content)
throw new Error(`${response.status} ${response.statusText}`)
}
return await response.text()
}
}
}

View File

@@ -35,10 +35,10 @@
"@xen-orchestra/qcow2": "^1.3.1",
"d3-time-format": "^4.1.0",
"golike-defer": "^0.5.1",
"http-request-plus": "^1.0.0",
"json-rpc-protocol": "^0.13.2",
"lodash": "^4.18.0",
"promise-toolbox": "^0.21.0",
"undici": "^6.26.0",
"vhd-lib": "^4.16.1",
"xo-common": "^0.10.0"
},

View File

@@ -1,6 +1,5 @@
import CancelToken from 'promise-toolbox/CancelToken'
import groupBy from 'lodash/groupBy.js'
import hrp from 'http-request-plus'
import ignoreErrors from 'promise-toolbox/ignoreErrors'
import pRetry from 'promise-toolbox/retry'
import pickBy from 'lodash/pickBy.js'
@@ -14,6 +13,7 @@ import { extract } from '@xen-orchestra/xapi/xoData.mjs'
import { finished } from 'node:stream'
import { incorrectState, forbiddenOperation } from 'xo-common/api-errors.js'
import { JsonRpcError } from 'json-rpc-protocol'
import { Agent } from 'undici'
import { Ref } from 'xen-api'
import isDefaultTemplate from './isDefaultTemplate.mjs'
@@ -21,6 +21,8 @@ import isVmRunning from './_isVmRunning.mjs'
const { warn, error } = createLogger('xo:xapi:vm')
const insecureAgent = new Agent({ connect: { rejectUnauthorized: false } })
const BIOS_STRINGS_KEYS = new Set([
'baseboard-asset-tag',
'baseboard-location-in-chassis',
@@ -192,11 +194,16 @@ class Vm {
}
try {
await hrp(url, {
const response = await fetch(url, {
headers,
rejectUnauthorized: false,
timeout: this._syncHookTimeout ?? 60e3,
dispatcher: insecureAgent,
signal: AbortSignal.timeout(this._syncHookTimeout ?? 60e3),
})
// the response body is not used, free the socket
await response.body?.cancel()
if (!response.ok) {
throw new Error(`${response.status} ${response.statusText}`)
}
} catch (error) {
warn('HTTP hook failed', { error, url, vm: uuid })
}

View File

@@ -40,11 +40,17 @@
<!--packages-start-->
- @xen-orchestra/acl minor
- @xen-orchestra/proxy-cli patch
- @xen-orchestra/rest-api minor
- @xen-orchestra/upload-ova patch
- @xen-orchestra/web minor
- @xen-orchestra/web-core minor
- @xen-orchestra/xapi patch
- xen-api minor
- xo-cli patch
- xo-common minor
- xo-server patch
- xo-server-ipmi-sensors minor
- xo-server-netbox patch
<!--packages-end-->

View File

@@ -1,7 +1,8 @@
import assert from 'assert'
import dns from 'dns'
import http from 'node:http'
import https from 'node:https'
import ms from 'ms'
import httpRequest from 'http-request-plus'
import map from 'lodash/map.js'
import noop from 'lodash/noop.js'
import Obfuscate from '@vates/obfuscate'
@@ -16,6 +17,7 @@ import { jsonHash } from '@vates/json-hash'
import { cancelable, defer, fromCallback, ignoreErrors, pDelay, pRetry, pTimeout } from 'promise-toolbox'
import { limitConcurrency } from 'limit-concurrency-decorator'
import { decorateClass } from '@vates/decorate-with'
import { pipeline } from 'node:stream'
import { ProxyAgent as HttpProxyAgent } from 'proxy-agent'
import getTaskResult from './_getTaskResult.mjs'
@@ -31,6 +33,95 @@ import { noSuchObject } from 'xo-common/api-errors.js'
const { debug } = createLogger('xen-api')
// ===================================================================
// Minimal `node:http(s)` request helper used by `putResource`.
// `fetch`/`undici` cannot be used here because `putResource` relies on a hack
// (see below) where a huge `content-length` is announced and the connection is
// cut once the real data has been sent
// Mimics the subset of `http-request-plus` that `putResource` depends on:
// resolves with the Node response on a 2xx status, otherwise rejects with an
// error carrying the response as `error.response` (whose body is drained, so a
// non-2xx response never leaks its socket). A stream body is never
// replayed, so redirects are not followed for it (`putResource` probes for
// redirections with an empty body beforehand).
function nodeRequest(url, { body, headers, maxRedirects = 5, signal, timeout, ...opts }) {
url = url instanceof URL ? url : new URL(url)
debug('nodeRequest', url.href)
const bodyIsStream = body != null && typeof body.pipe === 'function'
headers = { ...headers }
if (body !== undefined && headers['content-length'] === undefined) {
const length = bodyIsStream ? (body.headers?.['content-length'] ?? body.length) : Buffer.byteLength(body)
if (length !== undefined) {
headers['content-length'] = length
}
}
let redirectsLeft = bodyIsStream ? 0 : maxRedirects
const send = currentUrl =>
new Promise((resolve, reject) => {
const req = (currentUrl.protocol === 'https:' ? https : http).request(currentUrl, { ...opts, headers, signal })
// before the response is received, errors reject the promise; afterwards,
// they are forwarded to the response so an abnormally closed connection
// (the 1PiB hack in `putResource` cuts the socket once the data has been
// sent) does not reject a settled promise, and `putResource`'s
// post-processing can handle the `ERR_STREAM_PREMATURE_CLOSE` itself
let sendError = reject
const onError = error => sendError(error)
req.on('error', onError)
if (timeout !== undefined) {
req.setTimeout(timeout, () => {
const error = new Error('HTTP connection has timed out')
error.url = currentUrl.href
req.destroy(error)
})
}
req.on('response', response => {
const { statusCode } = response
const { location } = response.headers
const isRedirect = statusCode >= 300 && statusCode < 400
if (redirectsLeft > 0 && isRedirect && location !== undefined) {
--redirectsLeft
response.destroy()
resolve(send(new URL(location, currentUrl)))
return
}
sendError = error => response.destroy(error)
// a client `IncomingMessage` has no `.url`; expose the final URL so
// callers (e.g. `putResource`) can attach it as error context
response.url = currentUrl.href
const isOk = statusCode >= 200 && statusCode < 300
if (isOk) {
resolve(response)
} else {
const error = new Error(`${statusCode} ${response.statusMessage}`)
Object.defineProperty(error, 'response', { value: response })
response.destroy()
reject(error)
}
})
if (bodyIsStream) {
// let `pipeline` own the request errors while streaming
req.off('error', onError)
pipeline(body, req, error => {
if (error != null) {
sendError(error)
}
})
} else {
req.end(body)
}
})
return send(url)
}
// in seconds!
const EVENT_TIMEOUT = 60
@@ -562,7 +653,7 @@ export class Xapi extends EventEmitter {
await this._setHostAddressInUrl(url, host)
const doRequest = (url, opts) =>
httpRequest(url, {
nodeRequest(url, {
agent: this._httpAgent,
body,
headers,

View File

@@ -41,7 +41,6 @@
"@xen-orchestra/log": "^0.7.2",
"bind-property-descriptor": "^2.0.0",
"blocked": "^1.2.1",
"http-request-plus": "^1.0.2",
"jest-diff": "^29.0.3",
"json-rpc-protocol": "^0.13.1",
"limit-concurrency-decorator": "^0.6.0",

View File

@@ -1,14 +1,14 @@
#!/usr/bin/env node
import { Agent } from 'undici'
import { createReadStream, createWriteStream, readFileSync } from 'fs'
import { PassThrough, pipeline } from 'stream'
import { PassThrough, pipeline, Readable } from 'stream'
import { stat } from 'fs/promises'
import chalk from 'chalk'
import forEach from 'lodash/forEach.js'
import fromCallback from 'promise-toolbox/fromCallback'
import getKeys from 'lodash/keys.js'
import getopts from 'getopts'
import hrp from 'http-request-plus'
import identity from 'lodash/identity.js'
import isObject from 'lodash/isObject.js'
import micromatch from 'micromatch'
@@ -624,10 +624,12 @@ async function call(args) {
try {
// FIXME: do not use private properties.
const baseUrl = xo._url.replace(/^ws/, 'http')
const httpOptions = {
rejectUnauthorized: !(await getServerConfig()).allowUnauthorized,
timeout: 0,
}
const rejectUnauthorized = !(await getServerConfig()).allowUnauthorized
const dispatcher = new Agent({
connect: { rejectUnauthorized },
headersTimeout: 0,
bodyTimeout: 0,
})
const result = await xo.call(method, params)
let keys, key, url
@@ -638,9 +640,18 @@ async function call(args) {
ensurePathParam(method, file)
url = new URL(result[key], baseUrl)
const output = createOutputStream(file)
const response = await hrp(url, httpOptions)
const response = await fetch(url, { dispatcher })
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
return fromCallback(pipeline, response, streamStatsPrinter(response.headers['content-length']), output)
return fromCallback(
pipeline,
Readable.fromWeb(response.body),
streamStatsPrinter(response.headers.get('content-length')),
output
)
}
if (key === '$sendTo') {
@@ -650,14 +661,17 @@ async function call(args) {
const length = file === '-' ? undefined : (await stat(file)).size
const input = pipeline(file === '-' ? process.stdin : createReadStream(file), streamStatsPrinter(length), noop)
const response = await hrp(url, {
...httpOptions,
const response = await fetch(url, {
dispatcher,
body: input,
headers: length && {
'content-length': length,
},
duplex: 'half',
headers: length !== undefined ? { 'content-length': length } : {},
method: 'POST',
})
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
return response.text()
}
}

View File

@@ -34,7 +34,6 @@
"content-type": "^1.0.5",
"fs-extra": "^11.1.0",
"getopts": "^2.3.0",
"http-request-plus": "^1.0.0",
"human-format": "^1.0.0",
"lodash": "^4.18.0",
"micromatch": "^4.0.2",
@@ -43,6 +42,7 @@
"promise-toolbox": "^0.21.0",
"pw": "^0.0.4",
"split2": "^4.2.0",
"undici": "^6.26.0",
"xdg-basedir": "^5.1.0",
"xo-lib": "^0.11.2"
},

View File

@@ -1,13 +1,13 @@
import { basename, join } from 'node:path'
import { createReadStream, createWriteStream } from 'node:fs'
import { normalize } from 'node:path/posix'
import { Agent } from 'undici'
import { parse as parseContentType } from 'content-type'
import { pipeline } from 'node:stream'
import { pipeline, Readable } from 'node:stream'
import { pipeline as pPipeline } from 'node:stream/promises'
import { readChunk } from '@vates/read-chunk'
import { stat } from 'node:fs/promises'
import getopts from 'getopts'
import hrp from 'http-request-plus'
import merge from 'lodash/merge.js'
import set from 'lodash/set.js'
import split2 from 'split2'
@@ -74,10 +74,14 @@ const COMMANDS = {
output === '-'
? process.stdout
: createWriteStream(output.endsWith('/') ? join(output, basename(path)) : output, { flags: 'wx' })
return pPipeline(response, streamStatsPrinter(response.headers['content-length']), outputStream)
return pPipeline(
Readable.fromWeb(response.body),
streamStatsPrinter(response.headers.get('content-length')),
outputStream
)
}
const { type } = parseContentType(response)
const { type } = parseContentType(response.headers.get('content-type'))
if (type === 'application/json') {
const result = await response.json()
@@ -94,7 +98,7 @@ const COMMANDS = {
return this.json ? JSON.stringify(result, null, 2) : result
} else if (type === 'application/x-ndjson') {
const lines = pipeline(response, split2(), noop)
const lines = pipeline(Readable.fromWeb(response.body), split2(), noop)
let line
while ((line = await readChunk(lines)) !== null) {
const data = JSON.parse(line)
@@ -162,13 +166,14 @@ export async function rest(args) {
// FIXME: extract server parsing in dedicated module/function
const baseUrl = new Xo({ url: server })._url.replace(/^ws/, 'http')
const baseOpts = {
headers: {
cookie: 'authenticationToken=' + token,
},
rejectUnauthorized: !allowUnauthorized,
timeout: 0,
const baseHeaders = {
cookie: 'authenticationToken=' + token,
}
const dispatcher = new Agent({
connect: { rejectUnauthorized: !allowUnauthorized },
headersTimeout: 0,
bodyTimeout: 0,
})
if (command === undefined || !(command in COMMANDS)) {
return console.log('Available commands: ', Object.keys(COMMANDS).sort().join(', '))
@@ -195,17 +200,20 @@ export async function rest(args) {
}
}
try {
return await hrp(url, merge({}, baseOpts, opts))
} catch (error) {
const { response } = error
if (response === undefined) {
throw error
}
console.error(response.statusCode, response.statusMessage)
const merged = merge({ headers: { ...baseHeaders } }, opts)
const { body } = merged
const isBodyStream = body !== undefined && typeof body.pipe === 'function'
const duplex = isBodyStream ? { duplex: 'half' } : {}
const response = await fetch(url, {
...merged,
dispatcher,
...duplex,
})
if (!response.ok) {
console.error(response.status, response.statusText)
throw await response.text()
}
return response
},
json,
},

View File

@@ -165,35 +165,32 @@ class Netbox {
}
const httpRequest = async () => {
let response
let resBody = 'Netbox error could not be retrieved'
try {
response = await this.#xo.httpRequest(url, options)
if (Math.floor(response.statusCode / 100) === 2) {
resBody = await response.text()
if (resBody.length > 0) {
return JSON.parse(resBody)
}
return
const response = await this.#xo.httpRequest(url, options)
const resBody = await response.text()
if (resBody.length > 0) {
return JSON.parse(resBody)
}
const error = new Error(`${response.statusCode} ${response.statusMessage}`)
try {
resBody = await response.text()
error.netboxError = JSON.parse(resBody)
} catch (err) {
log.error(err)
// If the error couldn't be parsed, expose the response's raw body
error.netboxError = resBody
}
throw error
} catch (error) {
// root error won't have a body , most likely an error code line ECONRESET
error.method = method
error.requestBody = dataDebug
error.response = response
// On a non-2xx response, `httpRequest` throws with the (undrained)
// response attached as `error.response` consume its body to expose the Netbox error. A root error (e.g.
// ECONNRESET) has no response and is rethrown as-is.
const { response } = error
if (response !== undefined) {
let resBody = 'Netbox error could not be retrieved'
try {
resBody = await response.text()
error.netboxError = JSON.parse(resBody)
} catch (err) {
log.error(err)
// If the error couldn't be parsed, expose the response's raw body
error.netboxError = resBody
}
}
throw error
} finally {
response?.destroy()
}
}
@@ -249,7 +246,7 @@ class Netbox {
try {
this.#netboxVersion = semver.coerce((await this.#request('/status/'))['netbox-version']).version
} catch (err) {
if (err?.response?.statusCode === 404) {
if (err?.response?.status === 404) {
// Endpoint not supported on versions prior to v2.10
// Best effort to support earlier versions without knowing the version explicitly
return

View File

@@ -94,7 +94,6 @@
"helmet": "^3.9.0",
"highland": "^2.11.1",
"http-proxy": "^1.16.2",
"http-request-plus": "^1.0.3",
"@vates/http-server-plus": "^2.0.2",
"human-format": "^1.0.0",
"iterable-backoff": "^0.1.0",
@@ -140,6 +139,7 @@
"syslog-client": "^1.1.1",
"tar-stream": "^3.1.6",
"tmp": "^0.2.1",
"undici": "^6.26.0",
"unzipper": "^0.10.5",
"uuid": "^9.0.0",
"value-matcher": "^0.2.0",

View File

@@ -1,12 +1,11 @@
import * as multiparty from 'multiparty'
import assert from 'assert'
import hrp from 'http-request-plus'
import { createLogger } from '@xen-orchestra/log'
import { defer } from 'golike-defer'
import { format, JsonRpcError } from 'json-rpc-peer'
import { getStreamAsBuffer } from 'get-stream'
import { invalidParameters, noSuchObject } from 'xo-common/api-errors.js'
import { pipeline } from 'stream'
import { pipeline, Readable } from 'stream'
import { peekFooterFromVhdStream } from 'vhd-lib'
import { vmdkToVhd } from 'xo-vmdk-to-vhd'
@@ -305,9 +304,14 @@ async function importDisk({ sr, type, name, description, url, vmdkData }) {
throw invalidParameters('URL import is only compatible with VHD and raw formats')
}
const stream = await hrp(url)
const length = stream.headers['content-length']
if (length !== undefined) {
const response = await fetch(url)
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
const stream = Readable.fromWeb(response.body)
const length = response.headers.get('content-length')
if (length !== null) {
stream.length = length
}

View File

@@ -3,7 +3,7 @@ import { createLogger } from '@xen-orchestra/log'
import assert from 'assert'
import { format } from 'json-rpc-peer'
import { X509Certificate } from 'node:crypto'
import { pipeline } from 'node:stream'
import { pipeline, Readable } from 'node:stream'
import semver from 'semver'
import { incorrectState, invalidParameters } from 'xo-common/api-errors.js'
@@ -742,7 +742,7 @@ async function handleGetSystemStatus(req, res, { xapi, host }) {
// Download and stream to client
const response = await this.httpRequest(url, opts)
return fromCallback(pipeline, response, res)
return fromCallback(pipeline, Readable.fromWeb(response.body), res)
}
/**

View File

@@ -2,7 +2,7 @@ import TTLCache from '@isaacs/ttlcache'
import { asyncMap } from '@xen-orchestra/async-map'
import { createLogger } from '@xen-orchestra/log'
import { format } from 'json-rpc-peer'
import { pipeline } from 'node:stream'
import { pipeline, Readable } from 'node:stream'
import tarStream from 'tar-stream'
import { Ref } from 'xen-api'
import { incorrectState, invalidParameters } from 'xo-common/api-errors.js'
@@ -519,9 +519,8 @@ async function handleGetSystemStatuses(_req, res, { xapi, pool }) {
const filename = `${host.name_label}-system-status.tar.bz2`
// Get the size from Content-Length header if available
const size = response.headers['content-length']
? Number.parseInt(response.headers['content-length'], 10)
: undefined
const contentLength = response.headers.get('content-length')
const size = contentLength ? Number.parseInt(contentLength, 10) : undefined
if (size === undefined) {
throw new Error(`Missing Content-Length header for host ${host.name_label} system status download`)
@@ -539,7 +538,7 @@ async function handleGetSystemStatuses(_req, res, { xapi, pool }) {
}
})
await fromCallback(pipeline, response, entry)
await fromCallback(pipeline, Readable.fromWeb(response.body), entry)
}
// Finalize archive after all downloads complete

View File

@@ -6,7 +6,6 @@ import { asyncEach } from '@vates/async-each'
import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
import { Task } from '@xen-orchestra/mixins/Tasks.mjs'
import concat from 'lodash/concat.js'
import hrp from 'http-request-plus'
import mapKeys from 'lodash/mapKeys.js'
import { createLogger } from '@xen-orchestra/log'
import { defer } from 'golike-defer'
@@ -15,6 +14,8 @@ import { FAIL_ON_QUEUE } from 'limit-concurrency-decorator'
import { getStreamAsBuffer } from 'get-stream'
import { ignoreErrors, timeout } from 'promise-toolbox'
import { invalidParameters, noSuchObject, unauthorized } from 'xo-common/api-errors.js'
import { Agent } from 'undici'
import { Readable } from 'node:stream'
import { Ref } from 'xen-api'
import { forEach, map, mapFilter, noop, parseSize, safeDateFormat } from '../utils.mjs'
@@ -1367,8 +1368,19 @@ async function import_({ data, sr, type = 'xva', url }) {
}
const timeout = this.config.getOptionalDuration('jsonrpc-api.xvaImportFromUrlTimeout') ?? 6e3
const ref = await xapi.VM_import(await hrp(url, { timeout }), sr._xapiRef)
return xapi.call('VM.get_uuid', ref)
// `timeout` is an inactivity timeout: undici's headers/body timeouts match this semantic
const dispatcher = new Agent({ headersTimeout: timeout, bodyTimeout: timeout })
try {
const response = await fetch(url, { dispatcher })
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
const ref = await xapi.VM_import(Readable.fromWeb(response.body), sr._xapiRef)
return await xapi.call('VM.get_uuid', ref)
} finally {
await dispatcher.close()
}
}
return {

View File

@@ -6,6 +6,7 @@ import semver from 'semver'
import some from 'lodash/some.js'
import unzip from 'unzipper'
import { asyncEach } from '@vates/async-each'
import { Readable } from 'node:stream'
import { createLogger } from '@xen-orchestra/log'
import { decorateObject } from '@vates/decorate-with'
import { defer as deferrable } from 'golike-defer'
@@ -79,7 +80,7 @@ const methods = {
async _getXenUpdates() {
const response = await this.xo.httpRequest('https://updates.ops.xenserver.com/xenserver/updates.xml')
const data = parseXml(await response.buffer()).patchdata
const data = parseXml(Buffer.from(await response.arrayBuffer())).patchdata
const patches = { __proto__: null }
forEach(data.patches.patch, patch => {
@@ -463,7 +464,8 @@ const methods = {
}
const { username, apikey } = xsCredentials
let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
const response = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
let stream = Readable.fromWeb(response.body)
stream = await new Promise((resolve, reject) => {
const PATCH_RE = /\.xsupdate$/
stream
@@ -497,7 +499,8 @@ const methods = {
}
const { username, apikey } = xsCredentials
let stream = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
const response = await this.xo.httpRequest(patchInfo.url, { auth: `${username}:${apikey}` })
let stream = Readable.fromWeb(response.body)
stream = await new Promise((resolve, reject) => {
stream
.pipe(unzip.Parse())

View File

@@ -1,12 +1,20 @@
import hrp from 'http-request-plus'
import { EnvHttpProxyAgent } from 'undici'
import { ProxyAgent } from 'proxy-agent'
const { env } = process
export default class Http {
// automatically fetches the proxy to use for the requested URL from the environment
//
// still exposed as `httpAgent` and used as a Node HTTP agent by other parts of the app
_agent = new ProxyAgent()
// undici dispatchers for `fetch`, memoized by `${rejectUnauthorized}:${timeout}`
//
// `EnvHttpProxyAgent` reads the proxy configuration from the environment, kept
// up to date by `setHttpProxy`
_dispatchers = new Map()
// whether XO has a proxy set from its own config/environment
get hasOwnHttpProxy() {
return this._hasOwnHttpProxy
@@ -26,13 +34,54 @@ export default class Http {
})
}
async httpRequest(url, opts) {
try {
return await hrp(url, { ...opts, agent: this._agent })
} catch (error) {
error.response?.destroy()
_getDispatcher(rejectUnauthorized, timeout) {
const key = `${rejectUnauthorized}:${timeout}`
let dispatcher = this._dispatchers.get(key)
if (dispatcher === undefined) {
dispatcher = new EnvHttpProxyAgent({
connect: { rejectUnauthorized },
bodyTimeout: timeout,
headersTimeout: timeout,
})
this._dispatchers.set(key, dispatcher)
}
return dispatcher
}
// by default the failed response body is drained (freeing the socket) and its
// content exposed as `error.data`; pass `bypassStatusCheck: true` to instead
// take ownership of `error.response` and consume its body yourself
async httpRequest(
url,
{ auth, body, bypassStatusCheck = false, headers, rejectUnauthorized = true, timeout = 0, ...opts } = {}
) {
const finalHeaders = { ...headers }
if (auth !== undefined) {
finalHeaders.authorization = 'Basic ' + Buffer.from(auth).toString('base64')
}
const response = await fetch(url, {
...opts,
body,
headers: finalHeaders,
dispatcher: this._getDispatcher(rejectUnauthorized, timeout),
...(body !== undefined && typeof body.pipe === 'function' ? { duplex: 'half' } : {}),
})
if (!response.ok) {
const error = new Error(`${response.status} ${response.statusText}`)
error.response = response
if (!bypassStatusCheck) {
// drain the response body to free the socket
error.data = (await response.text().catch(() => '')).substring(0, 1024)
}
throw error
}
return response
}
// Inject the proxy into the environment, it will be automatically used by `_agent` and by most libs (e.g `axios`)
@@ -54,5 +103,12 @@ export default class Http {
env.no_proxy = env.NO_PROXY = noProxy
}
}
// the proxy configuration changed: drop cached dispatchers so they are rebuilt
// from the new environment
for (const dispatcher of this._dispatchers.values()) {
dispatcher.close().catch(() => {})
}
this._dispatchers.clear()
}
}

View File

@@ -1,13 +1,13 @@
import assert from 'assert'
import contentType from 'content-type'
import cookie from 'cookie'
import hrp from 'http-request-plus'
import isEmpty from 'lodash/isEmpty.js'
import omit from 'lodash/omit.js'
import parseSetCookie from 'set-cookie-parser'
import pumpify from 'pumpify'
import some from 'lodash/some.js'
import split2 from 'split2'
import { Agent } from 'undici'
import { compileTemplate } from '@xen-orchestra/template'
import { createLogger } from '@xen-orchestra/log'
import { decorateWith } from '@vates/decorate-with'
@@ -16,6 +16,7 @@ import { format, parse } from 'json-rpc-peer'
import { incorrectState, invalidParameters, noSuchObject } from 'xo-common/api-errors.js'
import { parseDuration } from '@vates/parse-duration'
import { readChunk, readChunkStrict } from '@vates/read-chunk'
import { Readable } from 'node:stream'
import { Ref } from 'xen-api'
import { synchronized } from 'decorator-synchronized'
import { timeout } from 'promise-toolbox'
@@ -75,6 +76,8 @@ async function populateProxy(proxy) {
export default class Proxy {
constructor(app) {
this._app = app
this._agents = new Map()
const rules = {
'{date}': (date = new Date()) => date.toISOString(),
}
@@ -100,6 +103,19 @@ export default class Proxy {
})
}
_getAgent(timeout) {
let agent = this._agents.get(timeout)
if (agent === undefined) {
agent = new Agent({
connect: { rejectUnauthorized: false },
headersTimeout: timeout,
bodyTimeout: timeout,
})
this._agents.set(timeout, agent)
}
return agent
}
async _getChannel() {
const PLACEHOLDER = '{xoChannel}'
let channel = this._app.config.get('xo-proxy.channel')
@@ -466,8 +482,7 @@ export default class Proxy {
Cookie: cookie.serialize('authenticationToken', proxy.authenticationToken),
},
method: 'POST',
rejectUnauthorized: false,
timeout,
dispatcher: this._getAgent(timeout),
}
if (proxy.address !== undefined) {
@@ -485,27 +500,33 @@ export default class Proxy {
url.hostname = address.includes(':') ? `[${address}]` : address
}
const response = await hrp(url, request)
const response = await fetch(url, request)
if (!response.ok) {
await response.body?.cancel() // free the socket
throw new Error(`${response.status} ${response.statusText}`)
}
const authenticationToken = parseSetCookie(response, {
const authenticationToken = parseSetCookie(response.headers.getSetCookie(), {
map: true,
}).authenticationToken?.value
if (authenticationToken !== undefined) {
await this.updateProxy(id, { authenticationToken })
}
const responseType = contentType.parse(response).type
const stream = Readable.fromWeb(response.body)
const responseType = contentType.parse(response.headers.get('content-type'))?.type
if (responseType === 'application/octet-stream') {
if (assertType !== 'stream') {
response.destroy()
stream.destroy()
throw new Error(`expect the result to be ${assertType}`)
}
return response
return stream
}
assert.strictEqual(responseType, 'application/json')
const lines = pumpify.obj(response, split2(JSON.parse))
const lines = pumpify.obj(stream, split2(JSON.parse))
const firstLine = await readChunk(lines)
const result = parse.result(firstLine)

View File

@@ -4690,14 +4690,6 @@
resolved "https://registry.yarnpkg.com/@vueuse/shared/-/shared-14.3.0.tgz#a3e7e6391f9ed7f363cbb28c32c4a278efaacbd0"
integrity sha512-bZpge9eSXwa4ToSiqJ7j6KRwhAsneMFoSz3LMWKQDkqimm3D/tbFlrklrs/IOqC8tEcYmXQZJ6N0UrjhBirVCg==
"@xen-orchestra/log@^0.6.0":
version "0.6.0"
resolved "https://registry.yarnpkg.com/@xen-orchestra/log/-/log-0.6.0.tgz#b7341818dbdc3a56facf815e7124867bf39ea89d"
integrity sha512-IA1J5LRfFZdb/TaPLG4oeZl1kApJW5BUWvmsr33eFXMX/KWS/FQt1Lzsl2OpH4AwNw8SpUCCl2+3wC5fBDtLlw==
dependencies:
lodash "^4.17.4"
promise-toolbox "^0.21.0"
"@xmldom/is-dom-node@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@xmldom/is-dom-node/-/is-dom-node-1.0.1.tgz#83b9f3e1260fb008061c6fa787b93a00f9be0629"
@@ -11144,13 +11136,6 @@ http-proxy@^1.16.2:
follow-redirects "^1.0.0"
requires-port "^1.0.0"
http-request-plus@^1.0.0, http-request-plus@^1.0.2, http-request-plus@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/http-request-plus/-/http-request-plus-1.0.3.tgz#34392950412b95056e05192fe7da15ec2c547c3b"
integrity sha512-566RDaiBFQ7xfEE01Kw+qjNSBD+a893gJArx/PtQg9Bex5TxGedav7nOOBJzB2/+ZbZ5mMt5aHdiju1sJB2qBw==
dependencies:
"@xen-orchestra/log" "^0.6.0"
http-server-plus@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/http-server-plus/-/http-server-plus-1.0.0.tgz#cb7adc1ca3e679e8728286a6c9b5b1bd012ccbfd"
@@ -19225,6 +19210,11 @@ undici@^6.2.1, undici@^6.5.0:
resolved "https://registry.yarnpkg.com/undici/-/undici-6.26.0.tgz#333a35b7f519c48d2dc6aeb38e4e91d9274e0652"
integrity sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A==
undici@^6.26.0:
version "6.27.0"
resolved "https://registry.yarnpkg.com/undici/-/undici-6.27.0.tgz#41f9e48f7c5a40d27376caaead8c9a9fc7bca9c4"
integrity sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==
undici@^7.8.0:
version "7.27.2"
resolved "https://registry.yarnpkg.com/undici/-/undici-7.27.2.tgz#f8fae968ee68377cfc61713d9cd152773716804f"