fix(openmetrics): keep the Prometheus secret across xo-server restarts

The `secret` property of `configurationSchema` used a random `default`. That
expression is re-evaluated every time the module is loaded, and xo-server never
persists the values ajv fills in from schema defaults, so each restart handed
the metrics endpoint a brand new bearer token and Prometheus started getting
401s.

Drop the default and generate the secret in `load()` instead, saving it through
`xo.configurePlugin()` so it survives a restart.

Introduced by #9323
See https://xcp-ng.org/forum/topic/12415
This commit is contained in:
Mathieu Piton
2026-08-24 10:16:19 +02:00
parent 9ef53890b8
commit fe29bd0d42
5 changed files with 89 additions and 7 deletions

View File

@@ -204,6 +204,8 @@ export type XoApp = {
backupGuard(poolId: XoPool['id']): Promise<void> backupGuard(poolId: XoPool['id']): Promise<void>
/* Throw if no authorization */ /* Throw if no authorization */
checkFeatureAuthorization(featureCode: FeatureCode): Promise<void> checkFeatureAuthorization(featureCode: FeatureCode): Promise<void>
/* validate, apply and persist the configuration of a plugin */
configurePlugin(id: string, configuration: unknown, mergeWithExisting?: boolean): Promise<void>
/* connect a server (XCP-ng/XenServer) */ /* connect a server (XCP-ng/XenServer) */
connectXenServer(id: XoServer['id']): Promise<void> connectXenServer(id: XoServer['id']): Promise<void>
// TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union) // TODO: replace all XoAclBasePrivilege with a more strict type. (discriminate union)

View File

@@ -73,6 +73,7 @@
- xo-server patch - xo-server patch
- xo-server-ipmi-sensors minor - xo-server-ipmi-sensors minor
- xo-server-netbox patch - xo-server-netbox patch
- xo-server-openmetrics patch
- xo-web patch - xo-web patch
<!--packages-end--> <!--packages-end-->

View File

@@ -403,14 +403,14 @@ The OpenMetrics plugin exposes a `/metrics` endpoint that Prometheus can scrape
2. Find and enable the **OpenMetrics** plugin. 2. Find and enable the **OpenMetrics** plugin.
3. Configure the following options: 3. Configure the following options:
| Option | Default | Description | | Option | Default | Description |
| --------------------- | ---------- | --------------------------------------------------------- | | --------------------- | ----------- | -------------------------------------------------------------------------- |
| **Prometheus secret** | (required) | Bearer token for authentication - you must set this value | | **Prometheus secret** | (generated) | Bearer token for authentication - generated on first load if left empty |
4. Save and load the plugin. 4. Save and load the plugin.
:::warning :::tip
You must set a **Prometheus secret** before loading the plugin. Without a valid secret, the metrics endpoint authentication is ineffective, potentially exposing sensitive infrastructure data (host resources, VM configurations, network details) to unauthorized access. Use a strong, random string (e.g., generated with `openssl rand -hex 32`). If you leave the **Prometheus secret** empty, one is generated the first time the plugin is loaded and saved in the plugin configuration, where you can read it back to configure Prometheus. You can also set your own strong, random string (e.g., generated with `openssl rand -hex 32`). Either way the secret is kept across xo-server restarts.
::: :::
### Prometheus Configuration ### Prometheus Configuration

View File

@@ -415,6 +415,9 @@ const __dirname = dirname(__filename)
const logger = createLogger('xo:xo-server-openmetrics') const logger = createLogger('xo:xo-server-openmetrics')
/** Id under which xo-server registers this plugin (directory name minus the `xo-server-` prefix) */
const PLUGIN_ID = 'openmetrics'
/** Default port for the OpenMetrics HTTP server */ /** Default port for the OpenMetrics HTTP server */
const DEFAULT_PORT = 9004 const DEFAULT_PORT = 9004
@@ -481,12 +484,36 @@ export const configurationSchema = {
type: 'string', type: 'string',
title: 'Prometheus secret', title: 'Prometheus secret',
description: 'Add this secret to http_config > authorization > credentials, and set type to Bearer', description: 'Add this secret to http_config > authorization > credentials, and set type to Bearer',
default: Buffer.from(getRandomValues(new Uint32Array(8))).toString('hex'),
}, },
}, },
additionalProperties: false, additionalProperties: false,
} }
/**
* Return the Prometheus bearer token, generating and persisting one on first use.
*
* The secret must survive an xo-server restart. It used to be a random
* `default` in `configurationSchema`: that expression is re-evaluated every
* time the module is loaded, and xo-server never saves the values it fills in
* from schema defaults, so each restart silently invalidated the token
* Prometheus was configured with.
*
* Exported for testability.
*/
export async function ensureSecret(
configuration: PluginConfiguration | undefined,
persist: (configuration: PluginConfiguration) => Promise<void>
): Promise<string> {
const secret = configuration?.secret
if (secret !== undefined && secret !== '') {
return secret
}
const generated = Buffer.from(getRandomValues(new Uint32Array(8))).toString('hex')
await persist({ secret: generated })
return generated
}
// ============================================================================ // ============================================================================
// XOSTOR helpers (module scope) // XOSTOR helpers (module scope)
// ============================================================================ // ============================================================================
@@ -779,11 +806,15 @@ class OpenMetricsPlugin {
return return
} }
const secret = await ensureSecret(this.#configuration, configuration =>
this.#xo.configurePlugin(PLUGIN_ID, configuration, true)
)
// Port and bindAddress are fixed for security (server is behind xo-server proxy) // Port and bindAddress are fixed for security (server is behind xo-server proxy)
const serverConfig: ServerConfiguration = { const serverConfig: ServerConfiguration = {
port: DEFAULT_PORT, port: DEFAULT_PORT,
bindAddress: DEFAULT_BIND_ADDRESS, bindAddress: DEFAULT_BIND_ADDRESS,
secret: this.#configuration?.secret ?? '', secret,
} }
logger.info('Starting OpenMetrics server', { logger.info('Starting OpenMetrics server', {

View File

@@ -0,0 +1,48 @@
/**
* Tests for the plugin configuration (Prometheus secret handling)
*/
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { configurationSchema, ensureSecret } from './index.mjs'
describe('configurationSchema', () => {
it('does not generate the secret as a schema default', () => {
// a `default` here is re-evaluated on every module load and never persisted
// by xo-server, which used to change the Prometheus secret at each restart
assert.equal((configurationSchema.properties.secret as Record<string, unknown>).default, undefined)
})
})
describe('ensureSecret', () => {
it('generates and persists a secret when none is configured', async () => {
const persisted: unknown[] = []
const secret = await ensureSecret(undefined, async configuration => {
persisted.push(configuration)
})
assert.match(secret, /^[0-9a-f]{16}$/)
assert.deepEqual(persisted, [{ secret }])
})
it('generates and persists a secret when the configured one is empty', async () => {
const persisted: unknown[] = []
const secret = await ensureSecret({ secret: '' }, async configuration => {
persisted.push(configuration)
})
assert.match(secret, /^[0-9a-f]{16}$/)
assert.deepEqual(persisted, [{ secret }])
})
it('keeps the configured secret and persists nothing', async () => {
const persisted: unknown[] = []
const secret = await ensureSecret({ secret: 'cafecafecafecafe' }, async configuration => {
persisted.push(configuration)
})
assert.equal(secret, 'cafecafecafecafe')
assert.deepEqual(persisted, [])
})
})