feat(openmetrics): add host uptime metric (#9449)

CHANGELOG.unreleased.md

@@ -19,6 +19,7 @@
 - [Plugins/load balancer] Add configurable VM migration cooldown to prevent oscillation (default 30min) (PR [#9388](https://github.com/vatesfr/xen-orchestra/pull/9388))
 - [REST API] Expose `POST /rest/v0/vms/:id/actions/migrate` to migrate a VM (PR [#9414](https://github.com/vatesfr/xen-orchestra/pull/9414))
 - [Netbox] Support version 4.5.x (PR [#9445](https://github.com/vatesfr/xen-orchestra/pull/9445))
+- [OpenMetrics] Add host uptime metric (`xcp_host_uptime_seconds`) (PR [#9449](https://github.com/vatesfr/xen-orchestra/pull/9449))
 
 ### Bug fixes
 

packages/xo-server-openmetrics/src/index.mts

@@ -73,6 +73,7 @@ export interface VmLabelInfo {
 export interface HostLabelInfo {
   name_label: string
   pifDeviceToNetworkName: Record<string, string> // { "eth0": "Management" }
+  startTime: number | null // Unix timestamp of host boot (from host.startTime)
 }
 
 export interface SrLabelInfo {
@@ -667,6 +668,7 @@ class OpenMetricsPlugin {
       labels.hosts[host.uuid] = {
         name_label: host.name_label,
         pifDeviceToNetworkName,
+        startTime: host.startTime,
       }
     }
 

packages/xo-server-openmetrics/src/open-metric-server.mts

@@ -23,6 +23,7 @@ import { parseRrdResponse, type ParsedRrdData } from './rrd-parser.mjs'
 import {
   formatAllPoolsToOpenMetrics,
   formatHostStatusMetrics,
+  formatHostUptimeMetrics,
   formatSrMetrics,
   formatToOpenMetrics,
   type HostStatusItem,
@@ -70,6 +71,7 @@ interface VmLabelInfo {
 interface HostLabelInfo {
   name_label: string
   pifDeviceToNetworkName: Record<string, string>
+  startTime: number | null
 }
 
 interface SrLabelInfo {
@@ -465,7 +467,12 @@ async function collectMetrics(): Promise<string> {
   const hostStatusOutput = hostStatusMetrics.length > 0 ? formatToOpenMetrics(hostStatusMetrics) : ''
   logger.debug('Formatted host status metrics', { hostCount: hostStatusMetrics.length })
 
-  // Combine pool metrics with RRD metrics, SR metrics, and host status metrics
+  // Format host uptime metrics
+  const uptimeMetrics = formatHostUptimeMetrics(credentials)
+  const uptimeMetricsOutput = uptimeMetrics.length > 0 ? formatToOpenMetrics(uptimeMetrics) : ''
+  logger.debug('Formatted host uptime metrics', { hostCount: uptimeMetrics.length })
+
+  // Combine pool metrics with RRD metrics, SR metrics, host status metrics, and uptime metrics
   // Remove the # EOF from rrdMetrics if present (we'll add our own)
   const rrdMetricsWithoutEof = rrdMetrics.replace(/\n# EOF$/, '')
 
@@ -483,6 +490,10 @@ async function collectMetrics(): Promise<string> {
     allMetricsSections.push(hostStatusOutput)
   }
 
+  if (uptimeMetricsOutput !== '') {
+    allMetricsSections.push(uptimeMetricsOutput)
+  }
+
   return allMetricsSections.join('\n') + '\n# EOF'
 }
 

packages/xo-server-openmetrics/src/openmetric-formatter.mts

@@ -938,3 +938,50 @@ export function formatHostStatusMetrics(hostStatusList: HostStatusItem[]): Forma
 
   return metrics
 }
+
+/**
+ * Format host uptime metrics to OpenMetrics format.
+ *
+ * Creates a FormattedMetric entry for each host's uptime, calculated as
+ * the difference between current time and host.startTime (boot time).
+ *
+ * @param labelContext - Label context containing host credentials and label lookup data
+ * @returns Array of FormattedMetric entries for host uptime
+ */
+export function formatHostUptimeMetrics(labelContext: LabelContext): FormattedMetric[] {
+  const metrics: FormattedMetric[] = []
+  const now = Math.floor(Date.now() / 1000)
+
+  for (const host of labelContext.hosts) {
+    const hostInfo = labelContext.labels.hosts[host.hostId]
+    if (hostInfo === undefined || hostInfo.startTime === null) {
+      continue
+    }
+
+    const uptimeSeconds = now - hostInfo.startTime
+
+    const labels: Record<string, string> = {
+      pool_id: host.poolId,
+      uuid: host.hostId,
+    }
+
+    if (host.poolLabel !== '') {
+      labels.pool_name = host.poolLabel
+    }
+
+    if (hostInfo.name_label !== '') {
+      labels.host_name = hostInfo.name_label
+    }
+
+    metrics.push({
+      name: `${METRIC_PREFIX}_host_uptime_seconds`,
+      help: 'Host uptime in seconds since boot',
+      type: 'gauge',
+      labels,
+      value: uptimeSeconds,
+      timestamp: now,
+    })
+  }
+
+  return metrics
+}

packages/xo-server-openmetrics/src/openmetric-formatter.test.mts

@@ -13,6 +13,7 @@ import {
   formatToOpenMetrics,
   formatAllPoolsToOpenMetrics,
   formatHostStatusMetrics,
+  formatHostUptimeMetrics,
   type FormattedMetric,
   type LabelContext,
 } from './openmetric-formatter.mjs'
@@ -543,6 +544,7 @@ describe('transformMetric with labelContext', () => {
         'host-uuid-123': {
           name_label: 'Host 1',
           pifDeviceToNetworkName: { eth0: 'Pool-wide network', eth1: 'Storage network' },
+          startTime: null,
         },
       },
       srs: {
@@ -854,6 +856,7 @@ describe('formatAllPoolsToOpenMetrics with labelContext', () => {
         'host-1': {
           name_label: 'Host 1',
           pifDeviceToNetworkName: { eth0: 'Management' },
+          startTime: null,
         },
       },
       srs: {},
@@ -1348,3 +1351,215 @@ describe('formatHostStatusMetrics', () => {
     assert.ok(output.includes('host_name="Host \\"with quotes\\""'))
   })
 })
+
+// ============================================================================
+// Host Uptime Metrics Tests
+// ============================================================================
+
+describe('formatHostUptimeMetrics', () => {
+  const createLabelContextWithUptime = (startTime: number | null): LabelContext => ({
+    hosts: [
+      {
+        hostId: 'host-uuid-123',
+        hostAddress: '192.168.1.1',
+        hostLabel: 'Host 1',
+        poolId: 'pool-456',
+        poolLabel: 'Production Pool',
+        sessionId: 'session-123',
+        protocol: 'https:',
+      },
+    ],
+    labels: {
+      vms: {},
+      hosts: {
+        'host-uuid-123': {
+          name_label: 'Host 1',
+          pifDeviceToNetworkName: {},
+          startTime,
+        },
+      },
+      srs: {},
+      srSuffixToUuid: {},
+      vdiUuidToSrUuid: {},
+    },
+  })
+
+  it('should generate uptime metric for host with valid startTime', () => {
+    const now = Math.floor(Date.now() / 1000)
+    const bootTime = now - 3600 // 1 hour ago
+    const labelContext = createLabelContextWithUptime(bootTime)
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 1)
+    const metric = metrics[0]!
+    assert.equal(metric.name, 'xcp_host_uptime_seconds')
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.help, 'Host uptime in seconds since boot')
+    assert.equal(metric.labels.pool_id, 'pool-456')
+    assert.equal(metric.labels.pool_name, 'Production Pool')
+    assert.equal(metric.labels.uuid, 'host-uuid-123')
+    assert.equal(metric.labels.host_name, 'Host 1')
+    // Value should be approximately 3600 (1 hour)
+    assert.ok(metric.value >= 3599 && metric.value <= 3601)
+  })
+
+  it('should skip host with null startTime', () => {
+    const labelContext = createLabelContextWithUptime(null)
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 0)
+  })
+
+  it('should skip host not found in labels', () => {
+    const labelContext: LabelContext = {
+      hosts: [
+        {
+          hostId: 'unknown-host',
+          hostAddress: '192.168.1.1',
+          hostLabel: 'Unknown Host',
+          poolId: 'pool-456',
+          poolLabel: 'Production',
+          sessionId: 'session-123',
+          protocol: 'https:',
+        },
+      ],
+      labels: {
+        vms: {},
+        hosts: {},
+        srs: {},
+        srSuffixToUuid: {},
+        vdiUuidToSrUuid: {},
+      },
+    }
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 0)
+  })
+
+  it('should generate metrics for multiple hosts', () => {
+    const now = Math.floor(Date.now() / 1000)
+    const labelContext: LabelContext = {
+      hosts: [
+        {
+          hostId: 'host-1',
+          hostAddress: '192.168.1.1',
+          hostLabel: 'Host 1',
+          poolId: 'pool-1',
+          poolLabel: 'Pool A',
+          sessionId: 'session-1',
+          protocol: 'https:',
+        },
+        {
+          hostId: 'host-2',
+          hostAddress: '192.168.1.2',
+          hostLabel: 'Host 2',
+          poolId: 'pool-1',
+          poolLabel: 'Pool A',
+          sessionId: 'session-1',
+          protocol: 'https:',
+        },
+      ],
+      labels: {
+        vms: {},
+        hosts: {
+          'host-1': {
+            name_label: 'Host 1',
+            pifDeviceToNetworkName: {},
+            startTime: now - 7200, // 2 hours
+          },
+          'host-2': {
+            name_label: 'Host 2',
+            pifDeviceToNetworkName: {},
+            startTime: now - 1800, // 30 minutes
+          },
+        },
+        srs: {},
+        srSuffixToUuid: {},
+        vdiUuidToSrUuid: {},
+      },
+    }
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 2)
+    const host1Metric = metrics.find(m => m.labels.uuid === 'host-1')
+    const host2Metric = metrics.find(m => m.labels.uuid === 'host-2')
+    assert.ok(host1Metric)
+    assert.ok(host2Metric)
+    assert.ok(host1Metric.value >= 7199 && host1Metric.value <= 7201)
+    assert.ok(host2Metric.value >= 1799 && host2Metric.value <= 1801)
+  })
+
+  it('should omit pool_name label when empty', () => {
+    const now = Math.floor(Date.now() / 1000)
+    const labelContext: LabelContext = {
+      hosts: [
+        {
+          hostId: 'host-1',
+          hostAddress: '192.168.1.1',
+          hostLabel: 'Host 1',
+          poolId: 'pool-1',
+          poolLabel: '', // Empty pool label
+          sessionId: 'session-1',
+          protocol: 'https:',
+        },
+      ],
+      labels: {
+        vms: {},
+        hosts: {
+          'host-1': {
+            name_label: 'Host 1',
+            pifDeviceToNetworkName: {},
+            startTime: now - 3600,
+          },
+        },
+        srs: {},
+        srSuffixToUuid: {},
+        vdiUuidToSrUuid: {},
+      },
+    }
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 1)
+    assert.equal(metrics[0]!.labels.pool_name, undefined)
+  })
+
+  it('should omit host_name label when empty', () => {
+    const now = Math.floor(Date.now() / 1000)
+    const labelContext: LabelContext = {
+      hosts: [
+        {
+          hostId: 'host-1',
+          hostAddress: '192.168.1.1',
+          hostLabel: 'Host 1',
+          poolId: 'pool-1',
+          poolLabel: 'Pool A',
+          sessionId: 'session-1',
+          protocol: 'https:',
+        },
+      ],
+      labels: {
+        vms: {},
+        hosts: {
+          'host-1': {
+            name_label: '', // Empty host name
+            pifDeviceToNetworkName: {},
+            startTime: now - 3600,
+          },
+        },
+        srs: {},
+        srSuffixToUuid: {},
+        vdiUuidToSrUuid: {},
+      },
+    }
+
+    const metrics = formatHostUptimeMetrics(labelContext)
+
+    assert.equal(metrics.length, 1)
+    assert.equal(metrics[0]!.labels.host_name, undefined)
+  })
+})